Filename: //lib//python3/////dist-packages/////debian/debtags.py

""" Facilities to work with debtags - tags for Debian packages """ # Copyright (C) 2006-2007 Enrico Zini <enrico@enricozini.org> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from __future__ import absolute_import, print_function import re import six try: import cPickle as pickle # type: ignore except ImportError: import pickle # type: ignore try: # pylint: disable=unused-import from typing import ( Callable, Dict, IO, Iterable, Iterator, List, Optional, Set, Text, Tuple, ) PkgTagDbType = Dict[str, Set[str]] TagPkgDbType = Dict[str, Set[str]] PkgFilterType = Callable[[Text], bool] TagFilterType = Callable[[Text], bool] PkgTagFilterType = Callable[[Tuple[Text, Set[Text]]], bool] except ImportError: # Lack of typing is not important at runtime pass from debian.deprecation import function_deprecated_by def parse_tags(input_data): # type: (Iterator[Text]) -> Iterator[Tuple[Set[str], Set[str]]] lre = re.compile(r"^(.+?)(?::?\s*|:\s+(.+?)\s*)$") for line in input_data: # Is there a way to remove the last character of a line that does not # make a copy of the entire line? m = lre.match(line) if not m: continue pkgs = set(m.group(1).split(', ')) if m.group(2): tags = set(m.group(2).split(', ')) else: tags = set() yield pkgs, tags parseTags = function_deprecated_by(parse_tags) def read_tag_database(input_data): # type: (Iterator[Text]) -> PkgTagDbType """Read the tag database, returning a pkg->tags dictionary""" db = {} # type: PkgTagDbType for pkgs, tags in parse_tags(input_data): # Create the tag set using the native set for p in pkgs: db[p] = tags.copy() return db readTagDatabase = function_deprecated_by(read_tag_database) def read_tag_database_reversed(input_data): # type: (Iterator[Text]) -> TagPkgDbType """Read the tag database, returning a tag->pkgs dictionary""" db = {} # type: TagPkgDbType for pkgs, tags in parse_tags(input_data): # Create the tag set using the native set for tag in tags: if tag in db: db[tag] |= pkgs else: db[tag] = pkgs.copy() return db readTagDatabaseReversed = function_deprecated_by(read_tag_database_reversed) def read_tag_database_both_ways( input_data, # type: Iterator[Text] tag_filter=None, # type: Optional[TagFilterType] ): # type: (...) -> Tuple[PkgTagDbType, TagPkgDbType] "Read the tag database, returning a pkg->tags and a tag->pkgs dictionary" db = {} # type: PkgTagDbType dbr = {} # type: TagPkgDbType for pkgs, tags in parse_tags(input_data): # Create the tag set using the native set if tag_filter is None: tags = set(tags) else: tags = set(filter(tag_filter, tags)) for pkg in pkgs: db[pkg] = tags.copy() for tag in tags: if tag in dbr: dbr[tag] |= pkgs else: dbr[tag] = pkgs.copy() return db, dbr readTagDatabaseBothWays = function_deprecated_by(read_tag_database_both_ways) def reverse(db): # type: (PkgTagDbType) -> TagPkgDbType """Reverse a tag database, from package -> tags to tag->packages""" res = {} # type: Dict[str, Set[str]] for pkg, tags in db.items(): for tag in tags: if tag not in res: res[tag] = set() res[tag].add(pkg) return res def output(db): # type: (PkgTagDbType) -> None "Write the tag database" for pkg, tags in db.items(): # Using % here seems awkward to me, but if I use calls to # sys.stdout.write it becomes a bit slower print("%s:" % (pkg), ", ".join(tags)) def relevance_index_function(full, sub): #type: ignore #return (float(sub.card(tag)) / float(sub.tag_count())) / \ # (float(full.card(tag)) / float(full.tag_count())) #return sub.card(tag) * full.card(tag) / sub.tag_count() # New cardinality divided by the old cardinality #return float(sub.card(tag)) / float(full.card(tag)) ## Same as before, but weighted by the relevance the tag had in the ## full collection, to downplay the importance of rare tags #return float(sub.card(tag) * full.card(tag)) / float(full.card(tag) * full.tag_count()) # Simplified version: # return float(sub.card(tag)) / float(full.tag_count()) # Weighted by the square root of the relevance, to downplay the very # common tags a bit # return lambda tag: float(sub.card(tag)) / float(full.card(tag)) * # math.sqrt(full.card(tag) / float(full.tag_count())) # return lambda tag: float(sub.card(tag)) / float(full.card(tag)) * # math.sqrt(full.card(tag) / float(full.package_count())) # One useless factor removed, and simplified further, thanks to Benjamin Mesing return lambda tag: float(sub.card(tag)**2) / float(full.card(tag)) # The difference between how many packages are in and how many packages are out # (problems: tags that mean many different things can be very much out # as well. In the case of 'image editor', for example, there will be # lots of editors not for images in the outside group. # It is very, very good for nonambiguous keywords like 'image'. # return lambda tag: 2 * sub.card(tag) - full.card(tag) # Same but it tries to downplay the 'how many are out' value in the # case of popular tags, to mitigate the 'there will always be popular # tags left out' cases. Does not seem to be much of an improvement. # return lambda tag: sub.card(tag) - float(full.card(tag) - sub.card(tag))/ # (math.sin(float(full.card(tag))*3.1415/full.package_count())/4 + 0.75) relevanceIndexFunction = function_deprecated_by(relevance_index_function) class DB: """ In-memory database mapping packages to tags and tags to packages. """ def __init__(self): # type: () -> None self.db = {} # type: PkgTagDbType self.rdb = {} # type: TagPkgDbType def read(self, input_data, # type: Iterator[Text] tag_filter=None, # type: Optional[TagFilterType] ): # type: (...) -> None """ Read the database from a file. Example:: # Read the system Debtags database db.read(open("/var/lib/debtags/package-tags", "r")) """ self.db, self.rdb = read_tag_database_both_ways(input_data, tag_filter) def qwrite(self, file): # type: (IO[bytes]) -> None """Quickly write the data to a pickled file""" pickle.dump(self.db, file) pickle.dump(self.rdb, file) def qread(self, file): # type: (IO[bytes]) -> None """Quickly read the data from a pickled file""" self.db = pickle.load(file) self.rdb = pickle.load(file) def insert(self, pkg, tags): # type: (str, Set[str]) -> None self.db[pkg] = tags.copy() for tag in tags: if tag in self.rdb: self.rdb[tag].add(pkg) else: self.rdb[tag] = set((pkg)) def dump(self): # type: () -> None output(self.db) def dump_reverse(self): # type: () -> None output(self.rdb) dumpReverse = function_deprecated_by(dump_reverse) def reverse(self): # type: () -> DB "Return the reverse collection, sharing tagsets with this one" res = DB() res.db = self.rdb res.rdb = self.db return res def facet_collection(self): # type: () -> DB """ Return a copy of this collection, but replaces the tag names with only their facets. """ fcoll = DB() tofacet = re.compile(r"^([^:]+).+") for pkg, tags in self.iter_packages_tags(): ftags = {tofacet.sub(r"\1", t) for t in tags} fcoll.insert(pkg, ftags) return fcoll facetCollection = function_deprecated_by(facet_collection) def copy(self): # type: () -> DB """ Return a copy of this collection, with the tagsets copied as well. """ res = DB() res.db = self.db.copy() res.rdb = self.rdb.copy() return res def reverse_copy(self): # type: () -> DB """ Return the reverse collection, with a copy of the tagsets of this one. """ res = DB() res.db = self.rdb.copy() res.rdb = self.db.copy() return res reverseCopy = function_deprecated_by(reverse_copy) def choose_packages(self, package_iter): # type: (Iterable[str]) -> DB """ Return a collection with only the packages in package_iter, sharing tagsets with this one """ res = DB() db = {} for pkg in package_iter: if pkg in self.db: db[pkg] = self.db[pkg] res.db = db res.rdb = reverse(db) return res choosePackages = function_deprecated_by(choose_packages) def choose_packages_copy(self, package_iter): # type: (Iterable[str]) -> DB """ Return a collection with only the packages in package_iter, with a copy of the tagsets of this one """ res = DB() db = {} for pkg in package_iter: db[pkg] = self.db[pkg] res.db = db res.rdb = reverse(db) return res choosePackagesCopy = function_deprecated_by(choose_packages_copy) def filter_packages(self, package_filter): # type: (PkgFilterType) -> DB """ Return a collection with only those packages that match a filter, sharing tagsets with this one. The filter will match on the package. """ res = DB() db = {} for pkg in filter(package_filter, six.iterkeys(self.db)): db[pkg] = self.db[pkg] res.db = db res.rdb = reverse(db) return res filterPackages = function_deprecated_by(filter_packages) def filter_packages_copy(self, filter_data): # type: (PkgFilterType) -> DB """ Return a collection with only those packages that match a filter, with a copy of the tagsets of this one. The filter will match on the package. """ res = DB() db = {} for pkg in filter(filter_data, six.iterkeys(self.db)): db[pkg] = self.db[pkg].copy() res.db = db res.rdb = reverse(db) return res filterPackagesCopy = function_deprecated_by(filter_packages_copy) def filter_packages_tags(self, package_tag_filter): # type: (PkgTagFilterType) -> DB """ Return a collection with only those packages that match a filter, sharing tagsets with this one. The filter will match on (package, tags). """ res = DB() db = {} for pkg, _ in filter(package_tag_filter, six.iteritems(self.db)): db[pkg] = self.db[pkg] res.db = db res.rdb = reverse(db) return res filterPackagesTags = function_deprecated_by(filter_packages_tags) def filter_packages_tags_copy(self, package_tag_filter): # type: (PkgTagFilterType) -> DB """ Return a collection with only those packages that match a filter, with a copy of the tagsets of this one. The filter will match on (package, tags). """ res = DB() db = {} for pkg, _ in filter(package_tag_filter, six.iteritems(self.db)): db[pkg] = self.db[pkg].copy() res.db = db res.rdb = reverse(db) return res filterPackagesTagsCopy = function_deprecated_by(filter_packages_tags_copy) def filter_tags(self, tag_filter): # type: (TagFilterType) -> DB """ Return a collection with only those tags that match a filter, sharing package sets with this one. The filter will match on the tag. """ res = DB() rdb = {} for tag in filter(tag_filter, six.iterkeys(self.rdb)): rdb[tag] = self.rdb[tag] res.rdb = rdb res.db = reverse(rdb) return res filterTags = function_deprecated_by(filter_tags) def filter_tags_copy(self, tag_filter): # type: (TagFilterType) -> DB """ Return a collection with only those tags that match a filter, with a copy of the package sets of this one. The filter will match on the tag. """ res = DB() rdb = {} for tag in filter(tag_filter, six.iterkeys(self.rdb)): rdb[tag] = self.rdb[tag].copy() res.rdb = rdb res.db = reverse(rdb) return res filterTagsCopy = function_deprecated_by(filter_tags_copy) def has_package(self, pkg): # type: (str) -> bool """Check if the collection contains the given package""" return pkg in self.db hasPackage = function_deprecated_by(has_package) def has_tag(self, tag): # type: (str) -> bool """Check if the collection contains packages tagged with tag""" return tag in self.rdb hasTag = function_deprecated_by(has_tag) def tags_of_package(self, pkg): # type: (str) -> Set[str] """Return the tag set of a package""" return self.db[pkg] if pkg in self.db else set() tagsOfPackage = function_deprecated_by(tags_of_package) def packages_of_tag(self, tag): # type: (str) -> Set[str] """Return the package set of a tag""" return self.rdb[tag] if tag in self.rdb else set() packagesOfTag = function_deprecated_by(packages_of_tag) def tags_of_packages(self, pkgs): # type: (Iterable[str]) -> Set[str] """Return the set of tags that have all the packages in ``pkgs``""" return set.union(*(self.tags_of_package(p) for p in pkgs)) tagsOfPackages = function_deprecated_by(tags_of_packages) def packages_of_tags(self, tags): # type: (Iterable[str]) -> Set[str] """Return the set of packages that have all the tags in ``tags``""" return set.union(*(self.packages_of_tag(t) for t in tags)) packagesOfTags = function_deprecated_by(packages_of_tags) def card(self, tag): # type: (str) -> int """ Return the cardinality of a tag """ return len(self.rdb[tag]) if tag in self.rdb else 0 def discriminance(self, tag): # type: (str) -> int """ Return the discriminance index if the tag. Th discriminance index of the tag is defined as the minimum number of packages that would be eliminated by selecting only those tagged with this tag or only those not tagged with this tag. """ n = self.card(tag) tot = self.package_count() return min(n, tot - n) def iter_packages(self): # type: () -> Iterator[str] """Iterate over the packages""" return six.iterkeys(self.db) iterPackages = function_deprecated_by(iter_packages) def iter_tags(self): # type: () -> Iterator[str] """Iterate over the tags""" return six.iterkeys(self.rdb) iterTags = function_deprecated_by(iter_tags) def iter_packages_tags(self): # type: () -> Iterator[Tuple[str, Set[str]]] """Iterate over 2-tuples of (pkg, tags)""" return six.iteritems(self.db) iterPackagesTags = function_deprecated_by(iter_packages_tags) def iter_tags_packages(self): # type: () -> Iterator[Tuple[str, Set[str]]] """Iterate over 2-tuples of (tag, pkgs)""" return six.iteritems(self.rdb) iterTagsPackages = function_deprecated_by(iter_tags_packages) def package_count(self): # type: () -> int """Return the number of packages""" return len(self.db) packageCount = function_deprecated_by(package_count) def tag_count(self): # type: () -> int """Return the number of tags""" return len(self.rdb) tagCount = function_deprecated_by(tag_count) def ideal_tagset(self, tags): # type: (List[str]) -> Set[str] """ Return an ideal selection of the top tags in a list of tags. Return the tagset made of the highest number of tags taken in consecutive sequence from the beginning of the given vector, that would intersect with the tagset of a comfortable amount of packages. Comfortable is defined in terms of how far it is from 7. """ # TODO: the scoring function is quite ok, but may need more # tuning. I also center it on 15 instead of 7 since we're # setting a starting point for the search, not a target point def score_fun(x): # type: (float) -> float return float((x-15)*(x-15))/x tagset = set() # type: Set[str] min_score = 3. for i in range(len(tags)): pkgs = self.packages_of_tags(tags[:i+1]) card = len(pkgs) if card == 0: break score = score_fun(card) if score < min_score: min_score = score tagset = set(tags[:i+1]) # Return always at least the first tag if not tagset: return set(tags[:1]) return tagset idealTagset = function_deprecated_by(ideal_tagset) def correlations(self): # type: () -> Iterator[Tuple[str, str, float]] """ Generate the list of correlation as a tuple (hastag, hasalsotag, score). Every tuple will indicate that the tag 'hastag' tends to also have 'hasalsotag' with a score of 'score'. """ for pivot in self.iter_tags(): # pylint: disable=cell-var-from-loop with_ = self.filter_packages_tags(lambda pt: pivot in pt[1]) without = self.filter_packages_tags(lambda pt: pivot not in pt[1]) for tag in with_.iter_tags(): if tag == pivot: continue has = float(with_.card(tag)) / float(with_.package_count()) hasnt = float(without.card(tag)) / float(without.package_count()) yield pivot, tag, has - hasnt

./Ninja\.