diff options
| author |   Matt Turner <mattst88@gentoo.org> | 2022-02-21 12:59:34 -0800 |
|---|---|---|
| committer | Matt Turner <mattst88@gentoo.org> | 2022-05-02 11:44:51 -0700 |
| commit | 3ec97d8f0cd427b7b9874b05f55d1aec5a1df8b0 (patch) | |
| tree | d6dc767b96cb791dbff24df54cc4ab126f19e225 | |
| parent | ebegin/eend: accept properly nested calls in different functions (diff) | |
| download | gentoo-portage-3ec97d8f0cd427b7b9874b05f55d1aec5a1df8b0.tar.xz gentoo-portage-3ec97d8f0cd427b7b9874b05f55d1aec5a1df8b0.zip | |
repoman: Remove
RepoMan sez: So long and thanks for all the fish! I'll be enjoying my
retirement now. Haven't I done enough to deserve that?
Bug: https://bugs.gentoo.org/835013
Signed-off-by: Matt Turner <mattst88@gentoo.org>
171 files changed, 32 insertions, 15135 deletions
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index d3c3fa3e1e1..1d6f1378c14 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -11,7 +11,7 @@ jobs: id: stragglers run: | echo "::set-output name=missed::$( - find bin repoman runtests -type f -not -name '*.py' -not -name '*.sh' | \ + find bin runtests -type f -not -name '*.py' -not -name '*.sh' | \ xargs grep -l '#!/usr/bin/env python' | tr $'\n' ' ')" - uses: psf/black@stable with: diff --git a/.gitignore b/.gitignore index a9f772c4c89..c04a22a0b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,3 @@ __pycache__/ /tags /.tox/ setup.cfg - -repoman/build diff --git a/cnf/make.globals b/cnf/make.globals index f42cffe8ba4..f951bb31724 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -147,7 +147,7 @@ PORTAGE_ELOG_MAILURI="root" PORTAGE_ELOG_MAILSUBJECT="[portage] ebuild log for \${PACKAGE} on \${HOST}" PORTAGE_ELOG_MAILFROM="portage@localhost" -# Signing command used by repoman +# Signing command used by egencache PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --digest-algo SHA256 --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\"" # btrfs.* attributes are irrelevant, see bug #527636. diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py index db42d836e1f..a1990411318 100644 --- a/lib/_emerge/Package.py +++ b/lib/_emerge/Package.py @@ -136,10 +136,7 @@ class Package(Task): # sync metadata with validated repo (may be UNKNOWN_REPO) self._metadata["repository"] = self.cpv.repo - if self.root_config.settings.local_config: - implicit_match = db._iuse_implicit_cnstr(self.cpv, self._metadata) - else: - implicit_match = db._repoman_iuse_implicit_cnstr(self.cpv, self._metadata) + implicit_match = db._iuse_implicit_cnstr(self.cpv, self._metadata) self.iuse = self._iuse( self, self._metadata["IUSE"].split(), implicit_match, self.eapi ) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index c3f872c4394..79fa1f3df61 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -9048,7 +9048,7 @@ class depgraph: ): # Make sure that portage always has all of its # RDEPENDs installed first, but ignore uninstalls - # (these occur when new portage blocks older repoman). + # (these occur when new portage blocks an older package version). return False selected_nodes.add(node) for child in mygraph.child_nodes(node, ignore_priority=ignore_priority): diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 43c3283e80d..f52329c90e1 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -219,18 +219,6 @@ class dbapi: yield cpv - def _repoman_iuse_implicit_cnstr(self, pkg, metadata): - """ - In repoman's version of _iuse_implicit_cnstr, account for modifications - of the self.settings reference between calls. - """ - eapi_attrs = _get_eapi_attrs(metadata["EAPI"]) - if eapi_attrs.iuse_effective: - iuse_implicit_match = lambda flag: self.settings._iuse_effective_match(flag) - else: - iuse_implicit_match = lambda flag: self.settings._iuse_implicit_match(flag) - return iuse_implicit_match - def _iuse_implicit_cnstr(self, pkg, metadata): """ Construct a callable that checks if a given USE flag should diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 93f3fee2fb2..cd919ba3134 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -359,9 +359,9 @@ class portdbapi(dbapi): def _set_porttrees(self, porttrees): """ - Consumers, such as repoman and emirrordist, may modify the porttrees - attribute in order to modify the effective set of repositories for - all portdbapi operations. + Consumers, such as emirrordist, may modify the porttrees attribute in + order to modify the effective set of repositories for all portdbapi + operations. @param porttrees: list of repo locations, in ascending order by repo priority diff --git a/lib/portage/tests/lazyimport/test_lazy_import_portage_baseline.py b/lib/portage/tests/lazyimport/test_lazy_import_portage_baseline.py index cf239240cea..8cc248db915 100644 --- a/lib/portage/tests/lazyimport/test_lazy_import_portage_baseline.py +++ b/lib/portage/tests/lazyimport/test_lazy_import_portage_baseline.py @@ -14,7 +14,7 @@ from _emerge.SpawnProcess import SpawnProcess class LazyImportPortageBaselineTestCase(TestCase): - _module_re = re.compile(r"^(portage|repoman|_emerge)\.") + _module_re = re.compile(r"^(portage|_emerge)\.") _baseline_imports = frozenset( [ diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py index 8811354b5bb..ec69ee06895 100644 --- a/lib/portage/tests/resolver/ResolverPlayground.py +++ b/lib/portage/tests/resolver/ResolverPlayground.py @@ -35,11 +35,6 @@ from _emerge.DependencyArg import DependencyArg from _emerge.depgraph import backtrack_depgraph from _emerge.RootConfig import RootConfig -try: - from repoman.tests import cnf_path_repoman -except ImportError: - cnf_path_repoman = None - class ResolverPlayground: """ @@ -664,11 +659,6 @@ class ResolverPlayground: for line in lines: f.write("%s\n" % line) - if cnf_path_repoman is not None: - # Create /usr/share/repoman - repoman_share_dir = os.path.join(self.eroot, "usr", "share", "repoman") - os.symlink(cnf_path_repoman, repoman_share_dir) - def _create_world(self, world, world_sets): # Create /var/lib/portage/world var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage") diff --git a/man/ebuild.5 b/man/ebuild.5 index 3a3982738cb..8402d667bd0 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -554,9 +554,9 @@ EAPI value then it will mask the package and refuse to perform any operations with it since this means that a newer version of portage needs to be installed first. For maximum backward compatiblity, a package should conform to the lowest possible EAPI. Note that anyone -who uses the \fBebuild\fR(1) and \fBrepoman\fR(1) commands with this -package will be required to have a version of portage that recognizes -the EAPI to which this package conforms. +who uses the \fBebuild\fR(1) command with this package will be +required to have a version of portage that recognizes the EAPI to +which this package conforms. .TP .B SRC_URI\fR = \fI"https://example.com/path/${P}.tar.gz" Contains a list of URIs for the required source files. It can contain diff --git a/man/make.conf.5 b/man/make.conf.5 index fc4a018c536..8c77eda2e39 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -457,9 +457,9 @@ enabled by default. .TP .B digest Autogenerate digests for packages when running the -\fBemerge\fR(1), \fBebuild\fR(1), or \fBrepoman\fR(1) commands. If -the \fIassume\-digests\fR feature is also enabled then existing SRC_URI digests -will be reused whenever they are available. +\fBemerge\fR(1) or \fBebuild\fR(1) commands. If the +\fIassume\-digests\fR feature is also enabled then existing SRC_URI +digests will be reused whenever they are available. .TP .B distcc Enable portage support for the distcc package. @@ -1051,7 +1051,7 @@ when \fBsign\fR is in \fBFEATURES\fR. In order to sign commits with to be configured by \fI`git config user.signingkey key_id`\fR. .TP .B PORTAGE_GPG_SIGNING_COMMAND -The command used by \fBrepoman\fR(1) to sign manifests when \fBsign\fR is +The command used by \fBegencache\fR(1) to sign manifests when \fBsign\fR is in \fBFEATURES\fR. .TP \fBPORTAGE_GRPNAME\fR = \fI[group]\fR diff --git a/man/portage.5 b/man/portage.5 index 50f20454c23..f42564664ca 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -944,14 +944,13 @@ Setting this attribute is generally not recommended since resulting changes in eclass inheritance may trigger performance issues due to invalidation of metadata cache. .br -When 'force = eclass\-overrides' attribute is not set, \fBegencache\fR(1), -\fBemirrordist\fR(1) and \fBrepoman\fR(1) ignore this attribute, -since operations performed by these tools are inherently -\fBnot\fR \fIsite\-specific\fR. +When 'force = eclass\-overrides' attribute is not set, \fBegencache\fR(1) +and \fBemirrordist\fR(1) ignore this attribute, since operations +performed by these tools are inherently \fBnot\fR \fIsite\-specific\fR. .TP .B force Specifies names of attributes, which should be forcefully respected by -\fBegencache\fR(1), \fBemirrordist\fR(1) and \fBrepoman\fR(1). +\fBegencache\fR(1) and \fBemirrordist\fR(1). .br Valid values: aliases, eclass\-overrides, masters .RE @@ -966,10 +965,9 @@ Setting this attribute is generally not recommended since resulting changes in eclass inheritance may trigger performance issues due to invalidation of metadata cache. .br -When 'force = aliases' attribute is not set, \fBegencache\fR(1), -\fBemirrordist\fR(1) and \fBrepoman\fR(1) ignore this attribute, -since operations performed by these tools are inherently -\fBnot\fR \fIsite\-specific\fR. +When 'force = aliases' attribute is not set, \fBegencache\fR(1) and +\fBemirrordist\fR(1) ignore this attribute, since operations performed by +these tools are inherently \fBnot\fR \fIsite\-specific\fR. .TP .B auto\-sync = yes|no|true|false This setting determines if the repo will be synced during "\fBemerge \-\-sync\fR" or @@ -990,14 +988,13 @@ Setting this attribute is generally not recommended since resulting changes in eclass inheritance may trigger performance issues due to invalidation of metadata cache. .br -When 'force = eclass\-overrides' attribute is not set, \fBegencache\fR(1), -\fBemirrordist\fR(1) and \fBrepoman\fR(1) ignore this attribute, -since operations performed by these tools are inherently -\fBnot\fR \fIsite\-specific\fR. +When 'force = eclass\-overrides' attribute is not set, \fBegencache\fR(1) +and \fBemirrordist\fR(1) ignore this attribute, since operations +performed by these tools are inherently \fBnot\fR \fIsite\-specific\fR. .TP .B force Specifies names of attributes, which should be forcefully respected by -\fBegencache\fR(1), \fBemirrordist\fR(1) and \fBrepoman\fR(1). +\fBegencache\fR(1) and \fBemirrordist\fR(1). .br Valid values: aliases, eclass\-overrides, masters .TP @@ -1011,10 +1008,9 @@ Setting this attribute is generally not recommended since resulting changes in eclass inheritance may trigger performance issues due to invalidation of metadata cache. .br -When 'force = masters' attribute is not set, \fBegencache\fR(1), -\fBemirrordist\fR(1) and \fBrepoman\fR(1) ignore this attribute, -since operations performed by these tools are inherently -\fBnot\fR \fIsite\-specific\fR. +When 'force = masters' attribute is not set, \fBegencache\fR(1) and +\fBemirrordist\fR(1) ignore this attribute, since operations performed by +these tools are inherently \fBnot\fR \fIsite\-specific\fR. .TP .B priority Specifies priority of given repository. @@ -1423,7 +1419,7 @@ Specifies information about the repository layout. \fISite-specific\fR overrides to \fBlayout.conf\fR settings may be specified in \fB/etc/portage/repos.conf\fR. Settings in \fBrepos.conf\fR take precedence over settings in -\fBlayout.conf\fR, except tools such as \fBrepoman\fR(1) and \fBegencache\fR(1) +\fBlayout.conf\fR, except tools such as \fBegencache\fR(1) ignore "aliases", "eclass-overrides" and "masters" attributes set in \fBrepos.conf\fR since their operations are inherently \fBnot\fR \fIsite\-specific\fR. @@ -1485,9 +1481,6 @@ here are not present in the Manifest, Portage will refetch all distfiles and update the respective entries to include them. Must be a subset of manifest\-hashes. If not specified, defaults to all manifest\-hashes. .TP -.BR update\-changelog " = [true|" false "]" -The default setting for repoman's --echangelog option. -.TP .BR cache\-formats " = [pms] [md5-dict]" The cache formats supported in the metadata tree. There is the old "pms" format and the newer/faster "md5-dict" format. Default is to detect dirs. @@ -1553,9 +1546,6 @@ use\-manifests = strict # customize the set of hashes generated for Manifest entries manifest\-hashes = SHA256 SHA512 WHIRLPOOL -# indicate that this repo enables repoman's --echangelog=y option automatically -update\-changelog = false - # indicate that this repo contains the md5-dict cache format, # which may be generated by egencache(1) cache\-formats = md5-dict @@ -1700,7 +1690,7 @@ package has been masked and WHO is doing the masking. .TP .BR profiles.desc List all the current stable and development profiles. If a profile is listed -here, then it will be checked by repoman. +here, then it will be checked by pkgcheck. .I Format: .nf \- comments begin with # (no inline comments) diff --git a/repoman/.repoman_not_installed b/repoman/.repoman_not_installed deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/.repoman_not_installed +++ /dev/null diff --git a/repoman/MANIFEST.in b/repoman/MANIFEST.in deleted file mode 100644 index d9ef2aee0b4..00000000000 --- a/repoman/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ - -# for the tests -include cnf/metadata.xsd -include .repoman_not_installed diff --git a/repoman/NEWS b/repoman/NEWS deleted file mode 100644 index c8d22d02498..00000000000 --- a/repoman/NEWS +++ /dev/null @@ -1,14 +0,0 @@ -News (mainly features/major bug fixes) - -repoman 2.3.8 -===================== -- Support for plugin module systems, see - https://wiki.gentoo.org/wiki/Project:Portage/Repoman-Module-specs - -repoman 2.3.0 -===================== -- Final release - -repoman 2.3.0_rc1 -===================== -- First test release diff --git a/repoman/README b/repoman/README deleted file mode 100644 index f0976e8414b..00000000000 --- a/repoman/README +++ /dev/null @@ -1,49 +0,0 @@ -About Portage -============= - -Portage is a package management system based on ports collections. The -Package Manager Specification Project (PMS) standardises and documents -the behaviour of Portage so that ebuild repositories can be used by other -package managers. - - -Dependencies -============ - -Python and Bash should be the only hard dependencies. Python 2.7 is the -minimum supported version. - - -Licensing and Legalese -======================= - -Portage is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -version 2 as published by the Free Software Foundation. - -Portage is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with Portage; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA -02110-1301, USA. - - -More information -================ - --DEVELOPING contains some code guidelines. --LICENSE contains the GNU General Public License version 2. --NEWS contains new features/major bug fixes for each version. --RELEASE NOTES contains mainly upgrade information for each version. --TEST-NOTES contains Portage unit test information. - - -Links -===== -Gentoo project page: <https://wiki.gentoo.org/wiki/Project:Portage> -PMS: <https://dev.gentoo.org/~ulm/pms/head/pms.html> -PMS git repo: <https://gitweb.gentoo.org/proj/pms.git/> diff --git a/repoman/RELEASE-NOTES b/repoman/RELEASE-NOTES deleted file mode 100644 index df34803ccb8..00000000000 --- a/repoman/RELEASE-NOTES +++ /dev/null @@ -1,213 +0,0 @@ -Release Notes; upgrade information mainly. -Features/major bugfixes are listed in NEWS - -repoman-3.0.3 -================================== -* Bug Fixes: - - Bug 608664 variable.phase check like pkgcheck VariableScopeCheck - - Bug 692486 Change message for preserve_old_lib - -repoman-3.0.2 -================================== -* Bug Fixes: - - Bug 748144 Make file.size fatal for files larger than 20 KiB - -repoman-3.0.1 -================================== -* Bug Fixes: - - Bug 737698 Update repoman for compatibility with portage-3.0.4. - -repoman-3.0.0 -================================== -* Bug Fixes: - - Bug 448462 Add --jobs and --load-average options which allow - dependency checks for multiple profiles to run in parallel. - -repoman-2.3.23 -================================== -* Bug Fixes: - - Bug 637824 Deprecate netsurf.eclass - -repoman-2.3.22 -================================== -* Bug Fixes: - - Bug 712106 only stage changes in index for commit mode - -repoman-2.3.21 -================================== -* Bug Fixes: - - Bug 712106 prevent spurious copyright header updates - - -repoman-2.3.20 -================================== -* Bug Fixes: - - Bug 541076 repoman commit: ignore unadded hidden files - - Bug 702100 repoman: support profiles/package.deprecated - - -repoman-2.3.19 -================================== -* Bug Fixes: - - Bug 667432 Rename DCO_SIGNED_OFF_BY config variable to SIGNED_OFF_BY. - - Bug 700456 Add --include-profiles=PROFILES - - -repoman-2.3.18 -================================== -* Bug Fixes: - - Bug 690786 Support metadata/layout.conf restrict-allowed - - Bug 699514 Detect dosym absolute paths starting with ${D}, ${ED} etc. - - Bug 699508 Fix unsafe string interpolation. - - -repoman-2.3.17 -================================== -* Bug Fixes: - - Bug 233589 Support PROPERTIES="live" - - -repoman-2.3.16 -================================== -* Bug Fixes: - - Bug 687420 Allow empty LICENSE in acct-* packages for GLEP 81 - - -repoman-2.3.15 -================================== -* Bug Fixes: - - Bug 688278 Accept 'Gentoo Foundation' copyright for old ebuilds - - -repoman-2.3.14 -================================== -* Bug Fixes: - - Bug 666330 Update for new copyright policy - - -repoman-2.3.13 -================================== -* Bug Fixes: - - Bug 686074 report error for unknown arguments - - -repoman-2.3.12 -================================== -* Bug Fixes: - - Bug 588752 fix exit code for manifest mode - - -repoman-2.3.11 -================================== -* Bug Fixes: - - Bug 666330 support new 'Gentoo Authors' copyright policy and - automatically apply the new policy when updating copyright headers. - Earlier versions of repoman will report an ebuild.badheader warning - for the new copyright headers. Users of earlier versions of repoman - should be advised to simply ignore the ebuild.badheader warning, or - else upgrade to the latest version of repoman. - - -repoman-2.3.10 -================================== -* Bug Fixes: - - Bug 649482 warn about = deps without revision - - Bug 660982 populate implicit IUSE for empty profile - - -repoman-2.3.8 -================================== -* The --experimental-repository-modules=<y|n> option enables a new - plugin modules system: - https://wiki.gentoo.org/wiki/Project:Portage/Repoman-Module-specs - - -repoman-2.3.7 -================================== -* Bug Fixes: - - Bug 552720 allow https for metadata.dtd URI - - -repoman-2.3.6 -================================== -* Bug Fixes: - - Bug 637460 Fix import paths for unit tests - - -repoman-2.3.5 -================================== -* Repository metadata/layout.conf supports manifest-required-hashes - attribute (default value is SHA512). - - -repoman-2.3.4 -================================== -* Support two new options: --bug (-b) and --closes (-c) - - -repoman-2.3.3 -================================== -* Bug Fixes: - - Bug 571546 Change how the tmp file for the commit msg is made -* flag URIs using http:// when https:// is available -* Add a check for relative dosym candidates -* Make all shebangs prefix friendly -* Mark ruby-2.0 as deprecated -* Detect inconsistent metadata.xml indentation -* Warn about dropped keywords only for latest in SLOT. - - -repoman-2.3.2 -================================== -* Bug Fixes: - - Bug 544938 add dev-qt/linguist-tools to RDEPEND.suspect set - - Bug 602002 make unused local USE flags an error - - Bug 533554 add HOMEPAGE.missingurischeme check - - Bug 610414 use regular expression to detect line continuations - - Bug 610954 Warn about stale CVS keywords in ebuild header - - Bug 611296 Don't update years in non-Gentoo copyright lines -* Add virtual/linuxtv-dvb-headers to RDEPEND.suspect set -* Add virtual/os-headers to RDEPEND.suspect set -* Add gentoo to remote-id type -* Define long for python3 Fixes: 006b168c1bb6 -* Commit footer improvements -* Check for empty files in filesdir -* Deprecate gpe.eclass, confutils.eclass - - -repoman-2.3.1 -================================== -* Bug Fixes: - - Bug 586864 skip QA checks in manifest mode - - Bug 591184 fix erroneous LICENSE.syntax - - use https to fetch metadata.xsd - - update links to https - - Make LIVEVCS.* checks fatal - - fix _depend_checks baddepsyntax accounting - - Fix commitmessage assignment not being done - - Add cat/pkg: substtution in the commitsgfile text - - Disable SRC_URI.mirror warnings when there is only 1 mirror - - Fix version output string to say Repoman, fix live versioning, - add portage version to --version output. - - Add repoman version to "Package manager:" commit footer - -repoman-2.3.0 -================================== -* Bug Fixes: - - Bug 584786 Add support for .git as a file when --separate-git-dir is used - - Bug 585864 fix missing vcs_files_to_cps repodir argument - - Bug 585388 fix KeyError during manifest generation -* New QA error: slot operator under '||' alternative - -repoman-2.3.0_rc1 -================================== -* Initial test release -* Bug Fixes: - - Bug 579460 Disable the $ID header check - - Bug 546010 Handle removed packages in vcs_files_to_cps - - Bug 581594 Fix commit 8e7971169c2 missing self in func() parameters - - Bug 581598 Add InvalidPackageName exception trap - - Bug 405017 Fix copyright update -* Includes the stage2 re-writes, the checks are now in a modular system - which is not yet fully plug-in capable. Becoming fully plug-in capable and - configurable is to be done in the stage3 rewites. diff --git a/repoman/TEST-NOTES b/repoman/TEST-NOTES deleted file mode 100644 index 8be5f9cf340..00000000000 --- a/repoman/TEST-NOTES +++ /dev/null @@ -1,45 +0,0 @@ -UnitTests ---------- - -Portage has some tests that use the unittest framework that ships with python (2.3-2.4ish) -Tests have a specific naming convention. - -in lib/portage/tests/ there is a runTest script that invokes lib/portage/tests/__init__.py - -This init looks at a hardcoded list of test dirs to search for tests. -If you add a new dir and don't see your new tests, make sure that the dir is in this list. - -On the subject of adding more directories; the layout is basically 1 directory per portage -file at this point (we have few files, and even fewer large files). Inside of the dir -you should have files of the form test_${function}.py. - -So if I was to write a vercmp test, and vercmp is in portage_versions. - -lib/portage/tests/portage_versions/test_vercmp.py - -would be the filename. - -The __init__.py file now does recursive tests, but you need to tell it so. For example, if -you had cache tests the dir format would be something like... - -lib/portage/tests/cache/flat_hash/test_foo.py - -and you would put "cache/flat_hash" into the testDirs variable in __init__.py. - - -Skipping --------- - -Please use the portage.tests.* classes as they support throwing a SkipException for -tests that are known to fail. Normally one uses testing to do Test Driven Development -(TDD); however we do not do that here. Therefore there are times when legitimate tests -exist but fail due to code in trunk. We would still like the suite to pass in some instances -because the suite is built around two things, testing functionality in the current code as -well as poking holes in the current code (isvalidatom is an example). So sometimes we desire -a test to point out that "this needs fixing" but it doesn't affect portage's overall -functionality. You should raise portage.tests.SkipException in that case. - -emerge ------- - -The emerge namespace currently has 0 tests (and no runner) diff --git a/repoman/bin/repoman b/repoman/bin/repoman deleted file mode 100755 index beaf8147a6a..00000000000 --- a/repoman/bin/repoman +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# Copyright 1999-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -"""Ebuild and tree health checks and maintenance utilities. -""" - -import sys -import errno - -# This block ensures that ^C interrupts are handled quietly. -try: - import signal - - def exithandler(signum, _frame): - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - sys.exit(128 + signum) - - signal.signal(signal.SIGINT, exithandler) - signal.signal(signal.SIGTERM, exithandler) - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - -except KeyboardInterrupt: - sys.exit(1) - -from os import path as osp - -here = osp.realpath(__file__) -if osp.isfile(osp.join(osp.dirname(osp.dirname(here)), ".repoman_not_installed")): - # Add the repoman subpkg - pym_path = osp.join(osp.dirname(osp.dirname(here)), "lib") - sys.path.insert(0, pym_path) - if osp.isfile( - osp.join(osp.dirname(osp.dirname(osp.dirname(here))), ".portage_not_installed") - ): - # Add the base portage pkg - pym_path = osp.join(osp.dirname(osp.dirname(osp.dirname(here))), "lib") - sys.path.insert(0, pym_path) - -import portage - -portage._internal_caller = True -from portage.util._eventloop.global_event_loop import global_event_loop -from repoman.main import repoman_main - -try: - sys.exit(repoman_main(sys.argv)) -except IOError as e: - if e.errno == errno.EACCES: - print("\nRepoman: Need user access") - sys.exit(1) - else: - raise -finally: - global_event_loop().close() diff --git a/repoman/cnf/linechecks/linechecks.yaml b/repoman/cnf/linechecks/linechecks.yaml deleted file mode 100644 index c6c72ab26ee..00000000000 --- a/repoman/cnf/linechecks/linechecks.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -# linecheck_help.yaml - -# Repoman API version (do not edit) -version: 1 -# minimum -repoman_version: 2.3.3 - -# configuration file for the LineCheck plugins run via the multicheck -# scan module -errors: - COPYRIGHT_ERROR: 'Invalid Copyright' - LICENSE_ERROR: 'Invalid Gentoo/GPL License' - ID_HEADER_ERROR: 'Stale CVS header' - NO_BLANK_LINE_ERROR: 'Non-blank line after header' - LEADING_SPACES_ERROR: 'Ebuild contains leading spaces' - TRAILING_WHITESPACE_ERROR: 'Trailing whitespace error' - READONLY_ASSIGNMENT_ERROR: 'Ebuild contains assignment to read-only variable' - MISSING_QUOTES_ERROR: 'Unquoted Variable' - NESTED_DIE_ERROR: 'Ebuild calls die in a subshell' - PATCHES_ERROR: 'PATCHES is not a bash array' - REDUNDANT_CD_S_ERROR: 'Ebuild has redundant cd ${S} statement' - EMAKE_PARALLEL_DISABLED: 'Upstream parallel compilation bug (ebuild calls emake -j1)' - EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS: 'Upstream parallel compilation bug (MAKEOPTS=-j1)' - DEPRECATED_BINDNOW_FLAGS: 'Deprecated bindnow-flags call' - EAPI_DEFINED_AFTER_INHERIT: 'EAPI defined after inherit' - NO_AS_NEEDED: 'Upstream asneeded linking bug (no-as-needed)' - PRESERVE_OLD_LIB: 'Ebuild calls preserve_old_lib function reserved for system packages' - BUILT_WITH_USE: 'built_with_use' - NO_OFFSET_WITH_HELPERS: 'Helper function is used with D, ROOT, ED, EROOT or EPREFIX' - USEQ_ERROR: 'Ebuild calls deprecated useq function' - HASQ_ERROR: 'Ebuild calls deprecated hasq function' - URI_HTTPS: 'Ebuild uses http:// but should use https://' - diff --git a/repoman/cnf/metadata.xsd b/repoman/cnf/metadata.xsd deleted file mode 100644 index 33ff58eac1b..00000000000 --- a/repoman/cnf/metadata.xsd +++ /dev/null @@ -1,548 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> - -<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'> - <!-- top-level variants --> - <xs:element name='pkgmetadata' type='pkgMetadataType'> - <!-- note: restrict uniquity rules are simplified - the spec says: one for each matched package - we can only do: one for each restrict rule --> - <xs:unique name='longDescUniquityConstraint'> - <xs:selector xpath='longdescription'/> - <xs:field xpath='@lang'/> - <xs:field xpath='@restrict'/> - </xs:unique> - <xs:unique name='maintainerUniquityConstraint'> - <xs:selector xpath='maintainer'/> - <xs:field xpath='email'/> - <xs:field xpath='@restrict'/> - </xs:unique> - <xs:unique name='slotsUniquityConstraint'> - <xs:selector xpath='slots'/> - <xs:field xpath='@lang'/> - </xs:unique> - <xs:unique name='upstreamSingleConstraint'> - <xs:selector xpath='upstream'/> - <xs:field xpath='@fake-only-once'/> - </xs:unique> - <xs:unique name='useUniquityConstraint'> - <xs:selector xpath='use'/> - <xs:field xpath='@lang'/> - </xs:unique> - </xs:element> - <xs:element name='catmetadata' type='catMetadataType'> - <xs:unique name='catLongDescUniquityConstraint'> - <xs:selector xpath='longdescription'/> - <xs:field xpath='@lang'/> - </xs:unique> - </xs:element> - - <!-- global elements --> - <xs:complexType name='pkgMetadataType'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='longdescription' type='longDescType'/> - <xs:element name='maintainer' type='maintainerType'> - <xs:unique name='maintainerDescUniquityConstraint'> - <xs:selector xpath='description'/> - <xs:field xpath='@lang'/> - </xs:unique> - </xs:element> - <xs:element name='slots' type='slotsType'> - <xs:unique name='slotUniquityConstraint'> - <xs:selector xpath='slot'/> - <xs:field xpath='@name'/> - </xs:unique> - <xs:unique name='subslotsSingleConstraint'> - <xs:selector xpath='subslots'/> - <xs:field xpath='@fake-only-once'/> - </xs:unique> - </xs:element> - <xs:element name='upstream' type='upstreamType'> - <xs:unique name='bugsToSingleConstraint'> - <xs:selector xpath='bugs-to'/> - <xs:field xpath='@fake-only-once'/> - </xs:unique> - <xs:unique name='changelogSingleConstraint'> - <xs:selector xpath='changelog'/> - <xs:field xpath='@fake-only-once'/> - </xs:unique> - <!-- prevent accidentally repeating the same remote --> - <xs:unique name='upstreamRemoteIdRepetitionConstraint'> - <xs:selector xpath='remote-id'/> - <xs:field xpath='@type'/> - <xs:field xpath='.'/> - </xs:unique> - <xs:unique name='upstreamDocUniquityConstraint'> - <xs:selector xpath='doc'/> - <xs:field xpath='@lang'/> - </xs:unique> - </xs:element> - <xs:element name='use' type='useType'> - <xs:unique name='flagUniquityConstraint'> - <xs:selector xpath='flag'/> - <xs:field xpath='@name'/> - <xs:field xpath='@restrict'/> - </xs:unique> - </xs:element> - </xs:choice> - </xs:complexType> - - <xs:complexType name='catMetadataType'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='longdescription' type='catLongDescType'/> - </xs:choice> - </xs:complexType> - - <!-- the huge <upstream/> structure --> - <xs:complexType name='upstreamType'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='maintainer' type='upstreamMaintainerType'> - <xs:unique name='upstreamMaintainerUniquityConstraint'> - <xs:selector xpath='maintainer'/> - <xs:field xpath='name'/> - </xs:unique> - </xs:element> - <xs:element name='changelog' type='urlOnceType'/> - <xs:element name='doc' type='upstreamDocType'/> - <xs:element name='bugs-to' type='urlOnceType'/> - <xs:element name='remote-id' type='upstreamRemoteIdType'/> - </xs:choice> - <xs:attribute name='fake-only-once' - fixed='there can be at most one <upstream/> element'/> - </xs:complexType> - - <!-- maintainer in two variants --> - <xs:complexType name='maintainerType'> - <xs:all> - <xs:element name='email' type='emailType' - minOccurs='1'/> - <xs:element name='name' type='xs:token' - minOccurs='0'/> - <xs:element name='description' type='maintainerDescType' - minOccurs='0'/> - </xs:all> - <xs:attribute name='type' type='maintainerTypeAttrType' - use='required'/> - <xs:attribute name='restrict' type='restrictAttrType'/> - </xs:complexType> - - <xs:simpleType name='maintainerTypeAttrType'> - <xs:restriction base='xs:token'> - <xs:enumeration value='person'/> - <xs:enumeration value='project'/> - </xs:restriction> - </xs:simpleType> - - <xs:complexType name='upstreamMaintainerType'> - <xs:all> - <xs:element name='email' type='emailType' - minOccurs='0'/> - <xs:element name='name' type='xs:token' - minOccurs='0'/> - </xs:all> - <xs:attribute name='status' type='upstreamMaintainerStatusAttrType' - default='unknown'/> - </xs:complexType> - - <xs:simpleType name='upstreamMaintainerStatusAttrType'> - <xs:restriction base='xs:token'> - <xs:enumeration value='active'/> - <xs:enumeration value='inactive'/> - <xs:enumeration value='unknown'/> - </xs:restriction> - </xs:simpleType> - - <xs:complexType name='maintainerDescType'> - <xs:simpleContent> - <xs:extension base="xs:token"> - <xs:attribute name='lang' type='langAttrType' default='en'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> - - <!-- long description --> - <xs:complexType name='longDescType' mixed='true'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='pkg' type='pkgType'/> - <xs:element name='cat' type='catType'/> - </xs:choice> - <xs:attribute name='lang' type='langAttrType' default='en'/> - <xs:attribute name='restrict' type='restrictAttrType'/> - </xs:complexType> - - <xs:complexType name='catLongDescType' mixed='true'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='pkg' type='pkgType'/> - <xs:element name='cat' type='catType'/> - </xs:choice> - <xs:attribute name='lang' type='langAttrType' default='en'/> - </xs:complexType> - - <!-- slots --> - <xs:complexType name='slotsType'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='slot' type='slotType'/> - <xs:element name='subslots' type='tokenOnceType'/> - </xs:choice> - <xs:attribute name='lang' type='langAttrType' default='en'/> - </xs:complexType> - - <xs:complexType name='slotType'> - <xs:simpleContent> - <xs:extension base="xs:token"> - <xs:attribute name='name' type='slotNameAttrType' - use='required'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> - - <xs:simpleType name='slotNameAttrType'> - <xs:restriction base='xs:token'> - <!-- PMS 3.1.3 Slot Names + special value '*' --> - <xs:pattern value="[A-Za-z0-9_][A-Za-z0-9+_.-]*|[*]"/> - </xs:restriction> - </xs:simpleType> - - <!-- use flags --> - <xs:complexType name='useType'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='flag' type='flagType'/> - </xs:choice> - <xs:attribute name='lang' type='langAttrType' default='en'/> - </xs:complexType> - - <xs:complexType name='flagType' mixed='true'> - <xs:choice minOccurs='0' maxOccurs='unbounded'> - <xs:element name='cat' type='catType'/> - <xs:element name='pkg' type='pkgType'/> - </xs:choice> - <xs:attribute name='name' type='flagNameAttrType' - use='required'/> - <xs:attribute name='restrict' type='restrictAttrType' - default=''/> - </xs:complexType> - - <xs:simpleType name='flagNameAttrType'> - <xs:restriction base='xs:token'> - <!-- PMS 3.1.4 USE Flag Names --> - <xs:pattern value="[A-Za-z0-9][A-Za-z0-9+_@-]*"/> - </xs:restriction> - </xs:simpleType> - - <!-- upstream-specific types --> - <xs:complexType name='upstreamDocType'> - <xs:simpleContent> - <xs:extension base="urlType"> - <xs:attribute name='lang' type='langAttrType' default='en'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> - - <xs:complexType name='upstreamRemoteIdType'> - <xs:simpleContent> - <xs:extension base="xs:token"> - <xs:attribute name='type' type='upstreamRemoteIdTypeAttrType' - use='required'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> - - <xs:simpleType name='upstreamRemoteIdTypeAttrType'> - <xs:restriction base='xs:token'> - <xs:enumeration value='bitbucket'/> - <xs:enumeration value='cpan'/> - <xs:enumeration value='cpan-module'/> - <xs:enumeration value='cpe'/> - <xs:enumeration value='cran'/> - <xs:enumeration value='ctan'/> - <xs:enumeration value='freecode'/> - <xs:enumeration value='freshmeat'/> - <xs:enumeration value='gentoo'/> - <xs:enumeration value='github'/> - <xs:enumeration value='gitlab'/> - <xs:enumeration value='gitorious'/> - <xs:enumeration value='google-code'/> - <xs:enumeration value='launchpad'/> - <xs:enumeration value='pear'/> - <xs:enumeration value='pecl'/> - <xs:enumeration value='pypi'/> - <xs:enumeration value='rubyforge'/> - <xs:enumeration value='rubygems'/> - <xs:enumeration value='sourceforge'/> - <xs:enumeration value='sourceforge-jp'/> - <xs:enumeration value='vim'/> - </xs:restriction> - </xs:simpleType> - - <!-- creepy mixed-text types --> - <xs:simpleType name='catType'> - <xs:restriction base='xs:token'> - <!-- PMS 3.1.1 Category Names --> - <xs:pattern value="[A-Za-z0-9_][A-Za-z0-9+_.-]*"/> - </xs:restriction> - </xs:simpleType> - <xs:simpleType name='pkgType'> - <xs:restriction base='xs:token'> - <!-- PMS 3.1.1 Category Names + 3.1.2 Package Names --> - <!-- note: this does not enforce the 'anything matching - the version syntax' requirement --> - <xs:pattern - value="[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_][A-Za-z0-9+_-]*"/> - </xs:restriction> - </xs:simpleType> - - <!-- common attributes --> - <xs:simpleType name='langAttrType'> - <xs:restriction base='xs:token'> - <!-- ISO 639-1 language codes --> - <xs:enumeration value='aa'/> - <xs:enumeration value='ab'/> - <xs:enumeration value='ae'/> - <xs:enumeration value='af'/> - <xs:enumeration value='ak'/> - <xs:enumeration value='am'/> - <xs:enumeration value='an'/> - <xs:enumeration value='ar'/> - <xs:enumeration value='as'/> - <xs:enumeration value='av'/> - <xs:enumeration value='ay'/> - <xs:enumeration value='az'/> - <xs:enumeration value='ba'/> - <xs:enumeration value='be'/> - <xs:enumeration value='bg'/> - <xs:enumeration value='bh'/> - <xs:enumeration value='bi'/> - <xs:enumeration value='bm'/> - <xs:enumeration value='bn'/> - <xs:enumeration value='bo'/> - <xs:enumeration value='bo'/> - <xs:enumeration value='br'/> - <xs:enumeration value='bs'/> - <xs:enumeration value='ca'/> - <xs:enumeration value='ce'/> - <xs:enumeration value='ch'/> - <xs:enumeration value='co'/> - <xs:enumeration value='cr'/> - <xs:enumeration value='cs'/> - <xs:enumeration value='cs'/> - <xs:enumeration value='cu'/> - <xs:enumeration value='cv'/> - <xs:enumeration value='cy'/> - <xs:enumeration value='cy'/> - <xs:enumeration value='da'/> - <xs:enumeration value='de'/> - <xs:enumeration value='de'/> - <xs:enumeration value='dv'/> - <xs:enumeration value='dz'/> - <xs:enumeration value='ee'/> - <xs:enumeration value='el'/> - <xs:enumeration value='el'/> - <xs:enumeration value='en'/> - <xs:enumeration value='eo'/> - <xs:enumeration value='es'/> - <xs:enumeration value='et'/> - <xs:enumeration value='eu'/> - <xs:enumeration value='eu'/> - <xs:enumeration value='fa'/> - <xs:enumeration value='fa'/> - <xs:enumeration value='ff'/> - <xs:enumeration value='fi'/> - <xs:enumeration value='fj'/> - <xs:enumeration value='fo'/> - <xs:enumeration value='fr'/> - <xs:enumeration value='fr'/> - <xs:enumeration value='fy'/> - <xs:enumeration value='ga'/> - <xs:enumeration value='ga'/> - <xs:enumeration value='Ga'/> - <xs:enumeration value='gd'/> - <xs:enumeration value='gl'/> - <xs:enumeration value='gn'/> - <xs:enumeration value='gu'/> - <xs:enumeration value='gv'/> - <xs:enumeration value='ha'/> - <xs:enumeration value='he'/> - <xs:enumeration value='hi'/> - <xs:enumeration value='ho'/> - <xs:enumeration value='hr'/> - <xs:enumeration value='ht'/> - <xs:enumeration value='hu'/> - <xs:enumeration value='hy'/> - <xs:enumeration value='hy'/> - <xs:enumeration value='hz'/> - <xs:enumeration value='ia'/> - <xs:enumeration value='id'/> - <xs:enumeration value='ie'/> - <xs:enumeration value='ig'/> - <xs:enumeration value='ii'/> - <xs:enumeration value='ik'/> - <xs:enumeration value='io'/> - <xs:enumeration value='is'/> - <xs:enumeration value='is'/> - <xs:enumeration value='it'/> - <xs:enumeration value='iu'/> - <xs:enumeration value='ja'/> - <xs:enumeration value='jv'/> - <xs:enumeration value='ka'/> - <xs:enumeration value='ka'/> - <xs:enumeration value='kg'/> - <xs:enumeration value='ki'/> - <xs:enumeration value='kj'/> - <xs:enumeration value='kk'/> - <xs:enumeration value='kl'/> - <xs:enumeration value='km'/> - <xs:enumeration value='kn'/> - <xs:enumeration value='ko'/> - <xs:enumeration value='kr'/> - <xs:enumeration value='ks'/> - <xs:enumeration value='ku'/> - <xs:enumeration value='kv'/> - <xs:enumeration value='kw'/> - <xs:enumeration value='ky'/> - <xs:enumeration value='la'/> - <xs:enumeration value='lb'/> - <xs:enumeration value='lg'/> - <xs:enumeration value='li'/> - <xs:enumeration value='ln'/> - <xs:enumeration value='lo'/> - <xs:enumeration value='lt'/> - <xs:enumeration value='lu'/> - <xs:enumeration value='lv'/> - <xs:enumeration value='mg'/> - <xs:enumeration value='mh'/> - <xs:enumeration value='mi'/> - <xs:enumeration value='mi'/> - <xs:enumeration value='mk'/> - <xs:enumeration value='mk'/> - <xs:enumeration value='ml'/> - <xs:enumeration value='mn'/> - <xs:enumeration value='mr'/> - <xs:enumeration value='ms'/> - <xs:enumeration value='ms'/> - <xs:enumeration value='mt'/> - <xs:enumeration value='my'/> - <xs:enumeration value='my'/> - <xs:enumeration value='na'/> - <xs:enumeration value='nb'/> - <xs:enumeration value='nd'/> - <xs:enumeration value='ne'/> - <xs:enumeration value='ng'/> - <xs:enumeration value='nl'/> - <xs:enumeration value='nl'/> - <xs:enumeration value='nn'/> - <xs:enumeration value='no'/> - <xs:enumeration value='nr'/> - <xs:enumeration value='nv'/> - <xs:enumeration value='ny'/> - <xs:enumeration value='oc'/> - <xs:enumeration value='oj'/> - <xs:enumeration value='om'/> - <xs:enumeration value='or'/> - <xs:enumeration value='os'/> - <xs:enumeration value='pa'/> - <xs:enumeration value='pi'/> - <xs:enumeration value='pl'/> - <xs:enumeration value='ps'/> - <xs:enumeration value='pt'/> - <xs:enumeration value='qu'/> - <xs:enumeration value='rm'/> - <xs:enumeration value='rn'/> - <xs:enumeration value='ro'/> - <xs:enumeration value='ro'/> - <xs:enumeration value='ru'/> - <xs:enumeration value='rw'/> - <xs:enumeration value='sa'/> - <xs:enumeration value='sc'/> - <xs:enumeration value='sd'/> - <xs:enumeration value='se'/> - <xs:enumeration value='sg'/> - <xs:enumeration value='si'/> - <xs:enumeration value='sk'/> - <xs:enumeration value='sk'/> - <xs:enumeration value='sl'/> - <xs:enumeration value='sm'/> - <xs:enumeration value='sn'/> - <xs:enumeration value='so'/> - <xs:enumeration value='sq'/> - <xs:enumeration value='sq'/> - <xs:enumeration value='sr'/> - <xs:enumeration value='ss'/> - <xs:enumeration value='st'/> - <xs:enumeration value='su'/> - <xs:enumeration value='sv'/> - <xs:enumeration value='sw'/> - <xs:enumeration value='ta'/> - <xs:enumeration value='te'/> - <xs:enumeration value='tg'/> - <xs:enumeration value='th'/> - <xs:enumeration value='ti'/> - <xs:enumeration value='tk'/> - <xs:enumeration value='tl'/> - <xs:enumeration value='tn'/> - <xs:enumeration value='to'/> - <xs:enumeration value='tr'/> - <xs:enumeration value='ts'/> - <xs:enumeration value='tt'/> - <xs:enumeration value='tw'/> - <xs:enumeration value='ty'/> - <xs:enumeration value='ug'/> - <xs:enumeration value='uk'/> - <xs:enumeration value='ur'/> - <xs:enumeration value='uz'/> - <xs:enumeration value='ve'/> - <xs:enumeration value='vi'/> - <xs:enumeration value='vo'/> - <xs:enumeration value='wa'/> - <xs:enumeration value='wo'/> - <xs:enumeration value='xh'/> - <xs:enumeration value='yi'/> - <xs:enumeration value='yo'/> - <xs:enumeration value='za'/> - <xs:enumeration value='zh'/> - <xs:enumeration value='zh'/> - <xs:enumeration value='zu'/> - </xs:restriction> - </xs:simpleType> - - <xs:simpleType name='restrictAttrType'> - <xs:restriction base='xs:token'> - <!-- simplified package dependency syntax --> - <!-- note: 'pure' package atom is technically valid too - but not really meaningful --> - <xs:pattern - value="(([<>]=?|[=~])[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_][A-Za-z0-9+_-]*-[0-9]+(\.[0-9]+)*[a-z]?((_alpha|_beta|_pre|_rc|_p)[0-9]*)*(-r[0-9]+)?\*?)?"/> - </xs:restriction> - </xs:simpleType> - - <!-- generic types --> - <xs:simpleType name='emailType'> - <xs:restriction base='xs:token'> - <!-- minimal safe regex --> - <xs:pattern value="[^@]+@[^.]+\..+"/> - </xs:restriction> - </xs:simpleType> - - <xs:complexType name='tokenOnceType'> - <xs:simpleContent> - <xs:extension base="xs:token"> - <xs:attribute name='fake-only-once' - fixed='there can be at most one element of this type'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> - - <xs:simpleType name='urlType'> - <xs:restriction base='xs:token'> - <!-- TODO: something better? --> - <xs:pattern value="(mailto:[^@]+@[^.]+\..+|https?://.+)"/> - </xs:restriction> - </xs:simpleType> - - <xs:complexType name='urlOnceType'> - <xs:simpleContent> - <xs:extension base="urlType"> - <xs:attribute name='fake-only-once' - fixed='there can be at most one element of this type'/> - </xs:extension> - </xs:simpleContent> - </xs:complexType> -</xs:schema> diff --git a/repoman/cnf/qa_data/qa_data.yaml b/repoman/cnf/qa_data/qa_data.yaml deleted file mode 100644 index 530c8c8061a..00000000000 --- a/repoman/cnf/qa_data/qa_data.yaml +++ /dev/null @@ -1,139 +0,0 @@ ---- -# qa_help.yaml - -# configuration file for the LineCheck plugins run via the multicheck -# scan module - -# Repoman API version (do not edit) -version: 1 -# minimum -repoman_version: 2.3.3 - -# qahelp: Primary QA help messages to describe the errors or warnings -# Dictionary -qahelp: - Entries: - IO_error: "Attempting to commit, and an IO error was encountered access the Entries file" - changelog: - ebuildadded: "An ebuild was added but the ChangeLog was not modified" - missing: "Missing ChangeLog files" - notadded: "ChangeLogs that exist but have not been added to cvs" - dependency: - bad: "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds)" - badmasked: "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds)" - badindev: "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch" - badmaskedindev: "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch" - badinexp: "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in experimental arch" - badmaskedinexp: "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in experimental arch" - badtilde: "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)" - deprecated: "Ebuild has a dependency that refers to a deprecated package" - equalsversion: "Suspicious =-dependency with a specific version and no rev. Please either use ~ if any revision is acceptable, or append -r0 to silence the warning." - missingslot: "RDEPEND matches more than one SLOT but does not specify a slot and/or use the := or :* slot operator" - perlcore: "This ebuild directly depends on a package in perl-core; it should use the corresponding virtual instead." - syntax: "Syntax error in dependency string (usually an extra/missing space/parenthesis)" - unknown: "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)" - badslotop: "RDEPEND contains ':=' slot operator under '||' dependency." - DESCRIPTION: - missing: "Ebuilds that have a missing or empty DESCRIPTION variable" - toolong: "DESCRIPTION is over %d characters" - digest: - assumed: "Existing digest must be assumed correct (Package level only)" - missing: "Some files listed in SRC_URI aren't referenced in the Manifest" - unused: "Some files listed in the Manifest aren't referenced in SRC_URI" - EAPI: - definition: "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)" - deprecated: "Ebuilds that use features that are deprecated in the current EAPI" - incompatible: "Ebuilds that use features that are only available with a different EAPI" - unsupported: "Ebuilds that have an unsupported EAPI version (you must upgrade portage)" - ebuild: - absdosym: "This ebuild uses absolute target to dosym where relative symlink could be used instead" - badheader: "This ebuild has a malformed header" - invalidname: "Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)" - majorsyn: "This ebuild has a major syntax error that may cause the ebuild to fail partially or fully" - minorsyn: "This ebuild has a minor syntax error that contravenes gentoo coding style" - namenomatch: "Ebuild files that do not have the same name as their parent directory" - notadded: "Ebuilds that exist but have not been added to the vcs" - nesteddie: "Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild." - output: "A simple sourcing of the ebuild produces output; this breaks ebuild policy." - patches: "PATCHES variable should be a bash array to ensure white space safety" - syntax: "Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure" - file: - executable: "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit" - size: "Files in the files directory must be under 20 KiB" - empty: "Empty file in the files directory" - name: "File/dir name must be composed of only the following chars: %s " - UTF8: "File is not UTF8 compliant" - HOMEPAGE: - missing: "Ebuilds that have a missing or empty HOMEPAGE variable" - virtual: "Virtuals that have a non-empty HOMEPAGE variable" - missingurischeme: "HOMEPAGE is missing an URI scheme" - inherit: - deprecated: "Ebuild inherits a deprecated eclass" - missing: "Ebuild uses functions from an eclass but does not inherit it" - unused: "Ebuild inherits an eclass but does not use it" - IUSE: - invalid: "This ebuild has a variable in IUSE that is not in the use.desc or its metadata.xml file" - missing: "This ebuild has a USE conditional which references a flag that is not listed in IUSE" - rubydeprecated: "The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore" - java: - eclassesnotused: "With virtual/jdk in DEPEND you must inherit a java eclass" - KEYWORDS: - dropped: "Ebuilds that appear to have dropped KEYWORDS for some arch" - invalid: "This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found" - missing: "Ebuilds that have a missing or empty KEYWORDS variable" - stable: "Ebuilds that have been added directly with stable KEYWORDS" - stupid: "Ebuilds that use KEYWORDS=-* instead of package.mask" - unsorted: "Ebuilds that contain KEYWORDS which are not sorted alphabetically." - LICENSE: - deprecated: "This ebuild is listing a deprecated license." - invalid: "This ebuild is listing a license that doesnt exist in portages license/ dir." - missing: "Ebuilds that have a missing or empty LICENSE variable" - syntax: "Syntax error in LICENSE (usually an extra/missing space/parenthesis)" - virtual: "Virtuals that have a non-empty LICENSE variable" - LIVEVCS: - stable: "This ebuild is a live checkout from a VCS but has stable keywords." - unmasked: "This ebuild is a live checkout from a VCS but has keywords and is not masked in the global package.mask." - manifest: - bad: "Manifest has missing or incorrect digests" - metadata: - bad: "Bad metadata.xml files" - missing: "Missing metadata.xml files" - warning: "Warnings in metadata.xml files" - PDEPEND: - suspect: "PDEPEND contains a package that usually only belongs in DEPEND." - portage: - internal: "The ebuild uses an internal Portage function or variable" - PROPERTIES: - syntax: "Syntax error in PROPERTIES (usually an extra/missing space/parenthesis)" - RDEPEND: - implicit: "RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4)" - suspect: "RDEPEND contains a package that usually only belongs in DEPEND." - repo: - eapi-banned: "The ebuild uses an EAPI which is banned by the repository's metadata/layout.conf settings" - eapi-deprecated: "The ebuild uses an EAPI which is deprecated by the repository's metadata/layout.conf settings" - RESTRICT: - invalid: "This ebuild contains invalid RESTRICT values." - syntax: "Syntax error in RESTRICT (usually an extra/missing space/parenthesis)" - REQUIRED_USE: - syntax: "Syntax error in REQUIRED_USE (usually an extra/missing space/parenthesis)" - SLOT: - invalid: "Ebuilds that have a missing or invalid SLOT variable value" - SRC_URI: - syntax: "Syntax error in SRC_URI (usually an extra/missing space/parenthesis)" - mirror: "A uri listed in profiles/thirdpartymirrors is found in SRC_URI" - upstream: - workaround: "The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org" - uri: - https: "URI uses http:// but should use https://" - usage: - obsolete: "The ebuild makes use of an obsolete construct" - variable: - invalidchar: "A variable contains an invalid character that is not part of the ASCII character set" - phase: "Variable referenced found within scope of incorrect ebuild phase as specified by PMS" - readonly: "Assigning a readonly variable" - usedwithhelpers: "Ebuild uses D, ROOT, BROOT, ED, EROOT or EPREFIX with helpers" - virtual: - suspect: "Ebuild contains a package that usually should be pulled via virtual/, not directly." - wxwidgets: - eclassnotused: "Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass" - diff --git a/repoman/cnf/repository/linechecks.yaml b/repoman/cnf/repository/linechecks.yaml deleted file mode 100644 index 528fdd99c42..00000000000 --- a/repoman/cnf/repository/linechecks.yaml +++ /dev/null @@ -1,248 +0,0 @@ ---- -# linecheck.yaml - -# configuration file for the LineCheck plugins run via the multicheck -# scan module -# no random drive-by commits please -# Please obtain authorization from the portage team -# -# Overlay maintainers override/add/negate checks at your discression -# but support for third party module will be limited to the plugin API -# - -# Repoman API version (do not edit) -version: 1 -# minimum -repoman_version: 2.3.3 - -eclass_export_functions: - - ant-tasks - - apache-2 - - apache-module - - aspell-dict - - autotools-utils - - base - - bsdmk - - cannadic - - clutter - - cmake-utils - - db - - distutils - - elisp - - embassy - - emboss - - emul-linux-x86 - - enlightenment - - font-ebdftopcf - - font - - fox - - freebsd - - freedict - - games - - games-ggz - - games-mods - - gdesklets - - gems - - gkrellm-plugin - - gnatbuild - - gnat - - gnome2 - - gnome-python-common - - gnustep-base - - go-mono - - gpe - - gst-plugins-bad - - gst-plugins-base - - gst-plugins-good - - gst-plugins-ugly - - gtk-sharp-module - - haskell-cabal - - horde - - java-ant-2 - - java-pkg-2 - - java-pkg-simple - - java-virtuals-2 - - kde4-base - - kde4-meta - - kernel-2 - - latex-package - - linux-mod - - mozlinguas - - myspell - - myspell-r2 - - mysql - - mysql-v2 - - mythtv-plugins - - oasis - - obs-service - - office-ext - - perl-app - - perl-module - - php-ext-base-r1 - - php-ext-pecl-r2 - - php-ext-source-r2 - - php-lib-r1 - - php-pear-lib-r1 - - php-pear-r1 - - python-distutils-ng - - python - - qt4-build - - qt4-r2 - - rox-0install - - rox - - ruby - - ruby-ng - - scsh - - selinux-policy-2 - - sgml-catalog - - stardict - - sword-module - - tetex-3 - - tetex - - texlive-module - - toolchain-binutils - - toolchain - - twisted - - vdr-plugin-2 - - vdr-plugin - - vim - - vim-plugin - - vim-spell - - virtuoso - - vmware - - vmware-mod - - waf-utils - - webapp - - xemacs-elisp - - xemacs-packages - - xfconf - - x-modular - - xorg-2 - - zproduct - -eclass_info_experimental_inherit: - autotools: - funcs: - - eaclocal - - eautoconf - - eautoheader - - eautomake - - eautoreconf - - _elibtoolize - - eautopoint - comprehensive: true - # Exempt eclasses: - # git - An EGIT_BOOTSTRAP variable may be used to call one of - # the autotools functions. - # subversion - An ESVN_BOOTSTRAP variable may be used to call one of - # the autotools functions. - exempt_eclasses: - - git - - git-2 - - subversion - - autotools-utils - eutils: - funcs: - - estack_push - - estack_pop - - eshopts_push - - eshopts_pop - - eumask_push - - eumask_pop - - epatch - - epatch_user - - emktemp - - edos2unix - - in_iuse - - use_if_iuse - - usex - comprehensive: false - flag-o-matic: - funcs: - - 'filter-(ld)?flags' - - 'strip-flags' - - 'strip-unsupported-flags' - - 'append-((ld|c(pp|xx)?))?flags' - - 'append-libs' - comprehensive: false - libtool: - funcs: - - elibtoolize - comprehensive: true - exempt_eclasses: - - autotools - multilib: - funcs: - - get_libdir - # These are "eclasses are the whole ebuild" type thing. - exempt_eclasses: - - autotools - - libtool - - multilib-minimal - comprehensive: false - multiprocessing: - funcs: - - makeopts_jobs - comprehensive: false - prefix: - funcs: - - eprefixify - comprehensive: true - toolchain-funcs: - funcs: - - gen_usr_ldscript - comprehensive: false - user: - funcs: - - enewuser - - enewgroup - - egetent - - egethome - - egetshell - - esethome - comprehensive: true - -# non experimental_inherit -eclass_info: - autotools: - funcs: - - eaclocal - - eautoconf - - eautoheader - - eautomake - - eautoreconf - - _elibtoolize - - eautopoint - comprehensive: true - ignore_missing: true - # Exempt eclasses: - # git - An EGIT_BOOTSTRAP variable may be used to call one of - # the autotools functions. - # subversion - An ESVN_BOOTSTRAP variable may be used to call one of - # the autotools functions. - exempt_eclasses: - - git - - git-2 - - subversion - - autotools-utils - prefix: - funcs: - - eprefixify - comprehensive: true - -usex_supported_eapis: - - "0" - - "1" - - "2" - - "3" - - "4" - - "4-slot-abi" - -in_iuse_supported_eapis: - - "0" - - "1" - - "2" - - "3" - - "4" - - "4-slot-abi" - - "5" diff --git a/repoman/cnf/repository/qa_data.yaml b/repoman/cnf/repository/qa_data.yaml deleted file mode 100644 index 2249000c3fe..00000000000 --- a/repoman/cnf/repository/qa_data.yaml +++ /dev/null @@ -1,163 +0,0 @@ ---- -# This yaml syntax file holds various configuration data for -# the Quality-Assurance checks performed. - -# no random drive-by commits please -# Please obtain authorization from the portage team -# -# Overlay maintainers override/add/negate checks at your discression -# but support for third party module will be limited to the plugin API -# - -# Repoman API version (do not edit) -version: 1 -# minimum -repoman_version: 2.3.3 - - -allowed_filename_chars: "a-zA-Z0-9._-+:" -max_description_length: 80 - -# missingvars check: Mandatory (non-defaulted) ebuild variables -# list -missingvars: - - KEYWORDS - - LICENSE - - DESCRIPTION - - HOMEPAGE - -# file.executable check, non executable files -# list -no_exec_files: - - Manifest - - ChangeLog - - metadata.xml - -# qawarnings: Non-fatal warnings, -# all values in here MUST have a corresponding qahelp entry -# list -qawarnings: - - changelog.missing - - changelog.notadded - - dependency.unknown - - dependency.badmasked - - dependency.badindev - - dependency.badmaskedindev - - dependency.badtilde - - dependency.deprecated - - dependency.equalsversion - - dependency.missingslot - - dependency.perlcore - - DESCRIPTION.toolong - - digest.assumed - - digest.unused - - EAPI.deprecated - - ebuild.notadded - - ebuild.nesteddie - - ebuild.absdosym - - ebuild.minorsyn - - ebuild.badheader - - ebuild.patches - - file.empty - - HOMEPAGE.virtual - - inherit.unused - - inherit.deprecated - - IUSE.rubydeprecated - - java.eclassesnotused - - KEYWORDS.dropped - - KEYWORDS.stupid - - KEYWORDS.missing - - KEYWORDS.unsorted - - LICENSE.deprecated - - LICENSE.virtual - - metadata.warning - - PDEPEND.suspect - - portage.internal - - RDEPEND.implicit - - RDEPEND.suspect - - repo.eapi-deprecated - - RESTRICT.invalid - - usage.obsolete - - upstream.workaround - - uri.https - - variable.phase - - virtual.suspect - - wxwidgets.eclassnotused - -# ruby_deprecated: Deprecated ruby targets -# list -ruby_deprecated: - - ruby_targets_ruby18 - - ruby_targets_ruby19 - - ruby_targets_ruby20 - -# suspect_rdepend: Common build only Dependencies -# not usually run time dependencies -# list -suspect_rdepend: - - app-arch/cabextract - - app-arch/rpm2targz - - app-doc/doxygen - - dev-lang/nasm - - dev-lang/swig - - dev-lang/yasm - - dev-perl/extutils-pkgconfig - - dev-qt/linguist-tools - - dev-util/byacc - - dev-util/cmake - - dev-util/ftjam - - dev-util/gperf - - dev-util/gtk-doc - - dev-util/gtk-doc-am - - dev-util/intltool - - dev-util/jam - - dev-util/pkg-config-lite - - dev-util/pkgconf - - dev-util/pkgconfig - - dev-util/pkgconfig-openbsd - - dev-util/scons - - dev-util/unifdef - - dev-util/yacc - - media-gfx/ebdftopcf - - sys-apps/help2man - - sys-devel/autoconf - - sys-devel/automake - - sys-devel/bin86 - - sys-devel/bison - - sys-devel/dev86 - - sys-devel/flex - - sys-devel/m4 - - sys-devel/pmake - - virtual/linux-sources - - virtual/linuxtv-dvb-headers - - virtual/os-headers - - virtual/pkgconfig - - x11-misc/bdftopcf - - x11-misc/imake - -# suspect_virtual: Dependencies that should usually be made to the virtual -# Not to the final target library -# dictionary -suspect_virtual: - dev-libs/libusb: virtual/libusb - dev-libs/libusb-compat: virtual/libusb - dev-libs/libusbx: virtual/libusb - dev-util/pkg-config-lite: virtual/pkgconfig - dev-util/pkgconf: virtual/pkgconfig - dev-util/pkgconfig: virtual/pkgconfig - dev-util/pkgconfig-openbsd: virtual/pkgconfig - -# valid_restrict: ??? -# list -valid_restrict: - - binchecks - - bindist - - fetch - - installsources - - mirror - - preserve-libs - - primaryuri - - splitdebug - - strip - - test - - userpriv diff --git a/repoman/cnf/repository/repository.yaml b/repoman/cnf/repository/repository.yaml deleted file mode 100644 index dbc1decaa76..00000000000 --- a/repoman/cnf/repository/repository.yaml +++ /dev/null @@ -1,76 +0,0 @@ ---- -# repository-modules.yaml -# -# This is the repository configuration file for repoman modules -# -# no random drive-by commits please -# Please obtain authorization from the portage team -# -# Overlay maintainers override/add/negate checks at your discression -# but support for third party module will be limited to the plugin API -# - -# Repoman API version (do not edit) -version: 1 -# minimum -repoman_version: 2.3.3 - -# NOTE: for non-gentoo repos, any custom modules added will need their -# module names to the modules list in order for them to run. - -# These are the non-mandatory modules that can be disabled/enabled. -# use -foo notation to disable, just like use flags -# Add custom modules to enable them too -scan_modules: - description - eapi - ebuild_metadata - fetches - files - keywords - live - manifests - multicheck - pkgmetadata - profile - restrict - ruby - -linechecks_modules: - assignment - eapi3assignment - implicitdepend - hasq - useq - preservelib - bindnow - inherit - dosym - definition - srcprepare - eapi3deprecated - pkgpretend - eapi4incompatible - eapi4gonevars - paralleldisabled - autodefault - gentooheader - nooffset - nesteddie - patches - emakeparallel - srccompileeconf - srcunpackpatches - pmsvariablerefphasescope - portageinternal - portageinternalvariableassignment - quote - quoteda - httpsuri - builtwith - uselesscds - uselessdodoc - whitespace - blankline - noasneeded - diff --git a/repoman/lib/repoman/__init__.py b/repoman/lib/repoman/__init__.py deleted file mode 100644 index b3690c07db9..00000000000 --- a/repoman/lib/repoman/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -import os.path -import subprocess -import sys -import time - -try: - import portage.const - import portage.proxy as proxy - from portage import _encodings, _shell_quote, _unicode_encode, _unicode_decode - from portage.const import PORTAGE_BASE_PATH, BASH_BINARY -except ImportError as e: - sys.stderr.write("\n\n") - sys.stderr.write( - "!!! Failed to complete portage imports. There are internal modules for\n" - ) - sys.stderr.write( - "!!! portage and failure here indicates that you have a problem with your\n" - ) - sys.stderr.write( - "!!! installation of portage. Please try a rescue portage located in the ebuild\n" - ) - sys.stderr.write( - "!!! repository under '/var/db/repos/gentoo/sys-apps/portage/files/' (default).\n" - ) - sys.stderr.write( - "!!! There is a README.RESCUE file that details the steps required to perform\n" - ) - sys.stderr.write("!!! a recovery of portage.\n") - sys.stderr.write(" " + str(e) + "\n\n") - raise - - -VERSION = "HEAD" - -REPOMAN_BASE_PATH = os.path.join( - os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3]) -) - -_not_installed = os.path.isfile( - os.path.join(REPOMAN_BASE_PATH, ".repoman_not_installed") -) - -if VERSION == "HEAD": - - class _LazyVersion(proxy.objectproxy.ObjectProxy): - def _get_target(self): - global VERSION - if VERSION is not self: - return VERSION - if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, ".git")): - encoding = _encodings["fs"] - cmd = [ - BASH_BINARY, - "-c", - ( - "cd %s ; git describe --match 'repoman-*' || exit $? ; " - + 'if [ -n "`git diff-index --name-only --diff-filter=M HEAD`" ] ; ' - + "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " - + "exit 0" - ) - % _shell_quote(PORTAGE_BASE_PATH), - ] - cmd = [ - _unicode_encode(x, encoding=encoding, errors="strict") for x in cmd - ] - proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - output = _unicode_decode(proc.communicate()[0], encoding=encoding) - status = proc.wait() - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: - output_lines = output.splitlines() - if output_lines: - version_split = output_lines[0].split("-") - if len(version_split) > 1: - VERSION = version_split[1] - patchlevel = False - if len(version_split) > 2: - patchlevel = True - VERSION = "%s_p%s" % (VERSION, version_split[2]) - if len(output_lines) > 1 and output_lines[1] == "modified": - head_timestamp = None - if len(output_lines) > 3: - try: - head_timestamp = int(output_lines[3]) - except ValueError: - pass - timestamp = int(time.time()) - if ( - head_timestamp is not None - and timestamp > head_timestamp - ): - timestamp = timestamp - head_timestamp - if not patchlevel: - VERSION = "%s_p0" % (VERSION,) - VERSION = "%s_p%d" % (VERSION, timestamp) - return VERSION - else: - print("NO output lines :(") - VERSION = "HEAD" - return VERSION - - VERSION = _LazyVersion() diff --git a/repoman/lib/repoman/_portage.py b/repoman/lib/repoman/_portage.py deleted file mode 100644 index 01a2a4cf525..00000000000 --- a/repoman/lib/repoman/_portage.py +++ /dev/null @@ -1,26 +0,0 @@ -"""repoman/_portage.py -Central location for the portage import. -There were problems when portage was imported by submodules -due to the portage instance was somehow different that the -initial portage import in main.py. The later portage imports -did not contain the repo it was working on. That repo was my cvs tree -and not listed in those subsequent portage imports. - -All modules should import portage from this one - -from repoman._portage import portage - -Then continue to import the remaining portage modules needed -""" - -import sys - -from os import path as osp - -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__)))) -sys.path.insert(0, pym_path) - -import portage - -portage._internal_caller = True -portage._disable_legacy_globals() diff --git a/repoman/lib/repoman/_subprocess.py b/repoman/lib/repoman/_subprocess.py deleted file mode 100644 index 08ee9e74848..00000000000 --- a/repoman/lib/repoman/_subprocess.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding:utf-8 -*- - - -import codecs -import subprocess - -# import our initialized portage instance -from repoman._portage import portage -from portage import _encodings, _unicode_encode - - -def repoman_getstatusoutput(cmd): - """ - Implements an interface similar to getstatusoutput(), but with - customized unicode handling (see bug #310789) and without the shell. - """ - args = portage.util.shlex_split(cmd) - - encoding = _encodings["fs"] - args = [_unicode_encode(x, encoding=encoding, errors="strict") for x in args] - proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = portage._unicode_decode( - proc.communicate()[0], encoding=encoding, errors="strict" - ) - if output and output[-1] == "\n": - # getstatusoutput strips one newline - output = output[:-1] - return (proc.wait(), output) - - -class repoman_popen(portage.proxy.objectproxy.ObjectProxy): - """ - Implements an interface similar to os.popen(), but with customized - unicode handling (see bug #310789) and without the shell. - """ - - __slots__ = ("_proc", "_stdout") - - def __init__(self, cmd): - args = portage.util.shlex_split(cmd) - - encoding = _encodings["fs"] - args = [_unicode_encode(x, encoding=encoding, errors="strict") for x in args] - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - object.__setattr__(self, "_proc", proc) - object.__setattr__( - self, "_stdout", codecs.getreader(encoding)(proc.stdout, "strict") - ) - - def _get_target(self): - return object.__getattribute__(self, "_stdout") - - __enter__ = _get_target - - def __exit__(self, exc_type, exc_value, traceback): - proc = object.__getattribute__(self, "_proc") - proc.wait() - proc.stdout.close() diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py deleted file mode 100644 index 920f0e97a65..00000000000 --- a/repoman/lib/repoman/actions.py +++ /dev/null @@ -1,828 +0,0 @@ -# -*- coding:utf-8 -*- -# Copyright 1999-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import errno -import io -import logging -import platform -import re -import shutil -import signal -import sys -import tempfile -import time -from itertools import chain - -try: - from urllib.parse import parse_qs, urlsplit, urlunsplit -except ImportError: - from urlparse import parse_qs, urlsplit, urlunsplit - -from _emerge.UserQuery import UserQuery - -from repoman._portage import portage -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.output import bold, create_color_func, green, red -from portage.package.ebuild.digestgen import digestgen -from portage.util import writemsg_level - -# pylint: disable=ungrouped-imports -from repoman.copyrights import update_copyright -from repoman.gpg import gpgsign, need_signature -from repoman import utilities -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman import VERSION - -bad = create_color_func("BAD") - - -class Actions: - """Handles post check result output and performs - the various vcs activities for committing the results""" - - def __init__(self, repo_settings, options, scanner, vcs_settings): - self.repo_settings = repo_settings - self.options = options - self.scanner = scanner - self.vcs_settings = vcs_settings - self.repoman_settings = repo_settings.repoman_settings - self.suggest = { - "ignore_masked": False, - "include_dev": False, - } - if scanner.have["pmasked"] and not ( - options.without_mask or options.ignore_masked - ): - self.suggest["ignore_masked"] = True - if scanner.have["dev_keywords"] and not options.include_dev: - self.suggest["include_dev"] = True - - def inform(self, can_force, result): - """Inform the user of all the problems found""" - if ( - self.suggest["ignore_masked"] or self.suggest["include_dev"] - ) and not self.options.quiet: - self._suggest() - if self.options.mode != "commit": - self._non_commit(result) - return False - else: - self._fail(result, can_force) - if self.options.pretend: - utilities.repoman_sez('"So, you want to play it safe. Good call."\n') - return True - - def perform(self, qa_output): - myautoadd = self._vcs_autoadd() - - self._vcs_deleted() - - changes = self.get_vcs_changed() - - mynew, mychanged, myremoved, no_expansion, expansion = changes - - # Manifests need to be regenerated after all other commits, so don't commit - # them now even if they have changed. - mymanifests = set() - myupdates = set() - for f in mychanged + mynew: - if "Manifest" == os.path.basename(f): - mymanifests.add(f) - else: - myupdates.add(f) - myupdates.difference_update(myremoved) - myupdates = list(myupdates) - mymanifests = list(mymanifests) - myheaders = [] - - commitmessage = self.options.commitmsg - if self.options.commitmsgfile: - try: - f = io.open( - _unicode_encode( - self.options.commitmsgfile, - encoding=_encodings["fs"], - errors="strict", - ), - mode="r", - encoding=_encodings["content"], - errors="replace", - ) - commitmessage = f.read() - f.close() - del f - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - portage.writemsg( - "!!! File Not Found:" - " --commitmsgfile='%s'\n" % self.options.commitmsgfile - ) - else: - raise - if commitmessage[:9].lower() in ("cat/pkg: ",): - commitmessage = self.msg_prefix() + commitmessage[9:] - - if commitmessage is not None and commitmessage.strip(): - res, expl = self.verify_commit_message(commitmessage) - if not res: - print(bad("RepoMan does not like your commit message:")) - print(expl) - if self.options.force: - print("(but proceeding due to --force)") - else: - sys.exit(1) - else: - commitmessage = None - msg_qa_output = qa_output - initial_message = None - while True: - commitmessage = self.get_new_commit_message( - msg_qa_output, commitmessage - ) - res, expl = self.verify_commit_message(commitmessage) - if res: - break - else: - full_expl = ( - """Issues with the commit message were found. Please fix it or remove -the whole commit message to abort. - -""" - + expl - ) - msg_qa_output = [ - " %s\n" % x for x in full_expl.splitlines() - ] + qa_output - - commitmessage = commitmessage.rstrip() - - # Update copyright for new and changed files - year = time.strftime("%Y", time.gmtime()) - updated_copyright = [] - for fn in chain(mynew, mychanged): - if fn.endswith(".diff") or fn.endswith(".patch"): - continue - if update_copyright(fn, year, pretend=self.options.pretend): - updated_copyright.append(fn) - - if updated_copyright and not ( - self.options.pretend or self.repo_settings.repo_config.thin_manifest - ): - for cp in sorted(self._vcs_files_to_cps(iter(updated_copyright))): - self._manifest_gen(cp) - - myupdates, broken_changelog_manifests = self.changelogs( - myupdates, - mymanifests, - myremoved, - mychanged, - myautoadd, - mynew, - commitmessage, - ) - - lines = commitmessage.splitlines() - lastline = lines[-1] - if len(lines) == 1 or re.match(r"^\S+:\s", lastline) is None: - commitmessage += "\n" - - commit_footer = self.get_commit_footer() - commitmessage += commit_footer - - print("* %s files being committed..." % green(str(len(myupdates))), end=" ") - - if not self.vcs_settings.needs_keyword_expansion: - # With some VCS types there's never any keyword expansion, so - # there's no need to regenerate manifests and all files will be - # committed in one big commit at the end. - logging.debug("VCS type doesn't need keyword expansion") - print() - elif not self.repo_settings.repo_config.thin_manifest: - logging.debug("perform: Calling thick_manifest()") - self.vcs_settings.changes.thick_manifest( - myupdates, myheaders, no_expansion, expansion - ) - - logging.info("myupdates: %s", myupdates) - logging.info("myheaders: %s", myheaders) - - uq = UserQuery(self.options) - if self.options.ask and uq.query("Commit changes?", True) != "Yes": - print("* aborting commit.") - sys.exit(128 + signal.SIGINT) - - # Handle the case where committed files have keywords which - # will change and need a priming commit before the Manifest - # can be committed. - if (myupdates or myremoved) and myheaders: - self.priming_commit(myupdates, myremoved, commitmessage) - - # When files are removed and re-added, the cvs server will put /Attic/ - # inside the $Header path. This code detects the problem and corrects it - # so that the Manifest will generate correctly. See bug #169500. - # Use binary mode in order to avoid potential character encoding issues. - self.vcs_settings.changes.clear_attic(myheaders) - - if self.scanner.repolevel == 1: - utilities.repoman_sez( - "\"You're rather crazy... " 'doing the entire repository."\n' - ) - - self.vcs_settings.changes.digest_regen( - myupdates, myremoved, mymanifests, self.scanner, broken_changelog_manifests - ) - - if self.repo_settings.sign_manifests: - self.sign_manifest(myupdates, myremoved, mymanifests) - - self.vcs_settings.changes.update_index(mymanifests, myupdates) - - self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage) - - if self.options.quiet: - return - print() - if self.vcs_settings.vcs: - print("Commit complete.") - else: - print( - "repoman was too scared" - " by not seeing any familiar version control file" - " that he forgot to commit anything" - ) - utilities.repoman_sez('"If everyone were like you, I\'d be out of business!"\n') - return - - def _vcs_files_to_cps(self, vcs_file_iter): - """ - Iterate over the given modified file paths returned from the vcs, - and return a frozenset containing category/pn strings for each - modified package. - - @param vcs_file_iter: file paths from vcs module - @type iter - @rtype: frozenset - @return: category/pn strings for each package. - """ - return vcs_files_to_cps( - vcs_file_iter, - self.repo_settings.repodir, - self.scanner.repolevel, - self.scanner.reposplit, - self.scanner.categories, - ) - - def _manifest_gen(self, cp): - """ - Generate manifest for a cp. - - @param cp: category/pn string - @type str - @rtype: bool - @return: True if successful, False otherwise - """ - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, cp) - return bool( - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) - ) - - def _suggest(self): - print() - if self.suggest["ignore_masked"]: - print( - bold( - "Note: use --without-mask to check " - "KEYWORDS on dependencies of masked packages" - ) - ) - - if self.suggest["include_dev"]: - print( - bold( - "Note: use --include-dev (-d) to check " - "dependencies for 'dev' profiles" - ) - ) - print() - - def _non_commit(self, result): - if result["full"]: - print(bold('Note: type "repoman full" for a complete listing.')) - if result["warn"] and not result["fail"]: - if self.options.quiet: - print(bold("Non-Fatal QA errors found")) - else: - utilities.repoman_sez( - "\"You're only giving me a partial QA payment?\n" - " I'll take it this time, but I'm not happy.\"" - ) - elif not result["fail"]: - if self.options.quiet: - print("No QA issues found") - else: - utilities.repoman_sez( - '"If everyone were like you, I\'d be out of business!"' - ) - elif result["fail"]: - print(bad("Please fix these important QA issues first.")) - if not self.options.quiet: - utilities.repoman_sez( - '"Make your QA payment on time' - " and you'll never see the likes of me.\"\n" - ) - - def _fail(self, result, can_force): - if ( - result["fail"] - and can_force - and self.options.force - and not self.options.pretend - ): - utilities.repoman_sez( - ' "You want to commit even with these QA issues?\n' - " I'll take it this time, but I'm not happy.\"\n" - ) - elif result["fail"]: - if self.options.force and not can_force: - print( - bad( - "The --force option has been disabled" - " due to extraordinary issues." - ) - ) - print(bad("Please fix these important QA issues first.")) - utilities.repoman_sez( - '"Make your QA payment on time' - " and you'll never see the likes of me.\"\n" - ) - sys.exit(1) - - def _vcs_autoadd(self): - myunadded = self.vcs_settings.changes.unadded - myautoadd = [] - if myunadded: - for x in range(len(myunadded) - 1, -1, -1): - xs = myunadded[x].split("/") - if ( - any(token.startswith(".") and token != "." for token in xs) - or self.repo_settings.repo_config.find_invalid_path_char( - myunadded[x] - ) - != -1 - ): - # The Manifest excludes this file, - # so it's safe to ignore. - del myunadded[x] - elif xs[-1] == "files": - print("!!! files dir is not added! Please correct this.") - sys.exit(-1) - elif xs[-1] == "Manifest": - # It's a manifest... auto add - myautoadd += [myunadded[x]] - del myunadded[x] - - if myunadded: - print( - red( - "!!! The following files are in your local tree" - " but are not added to the master" - ) - ) - print( - red( - "!!! tree. Please remove them from the local tree" - " or add them to the master tree." - ) - ) - for x in myunadded: - print(" ", x) - print() - print() - sys.exit(1) - return myautoadd - - def _vcs_deleted(self): - if self.vcs_settings.changes.has_deleted: - print( - red( - "!!! The following files are removed manually" - " from your local tree but are not" - ) - ) - print( - red( - "!!! removed from the repository." - ' Please remove them, using "%s remove [FILES]".' - % self.vcs_settings.vcs - ) - ) - for x in self.vcs_settings.changes.deleted: - print(" ", x) - print() - print() - sys.exit(1) - - def get_vcs_changed(self): - """Holding function which calls the approriate VCS module for the data""" - changes = self.vcs_settings.changes - # re-run the scan to pick up a newly modified Manifest file - logging.debug("RE-scanning for changes...") - changes.scan() - - if not changes.has_changes: - utilities.repoman_sez('"Doing nothing is not always good for QA."') - print() - print("(Didn't find any changed files...)") - print() - sys.exit(1) - return ( - changes.new, - changes.changed, - changes.removed, - changes.no_expansion, - changes.expansion, - ) - - https_bugtrackers = frozenset( - [ - "bitbucket.org", - "bugs.gentoo.org", - "github.com", - "gitlab.com", - ] - ) - - def get_commit_footer(self): - portage_version = getattr(portage, "VERSION", None) - gpg_key = self.repoman_settings.get("PORTAGE_GPG_KEY", "") - signoff = self.repoman_settings.get("SIGNED_OFF_BY", "") - report_options = [] - if self.options.force: - report_options.append("--force") - if self.options.ignore_arches: - report_options.append("--ignore-arches") - if self.scanner.include_arches is not None: - report_options.append( - '--include-arches="%s"' % " ".join(sorted(self.scanner.include_arches)) - ) - if self.scanner.include_profiles is not None: - report_options.append( - '--include-profiles="%s"' - % " ".join(sorted(self.scanner.include_profiles)) - ) - - if portage_version is None: - sys.stderr.write("Failed to insert portage version in message!\n") - sys.stderr.flush() - portage_version = "Unknown" - - # Common part of commit footer - commit_footer = "" - for tag, bug in chain( - (("Bug", x) for x in self.options.bug), - (("Closes", x) for x in self.options.closes), - ): - # case 1: pure number NNNNNN - if bug.isdigit(): - bug = "https://bugs.gentoo.org/%s" % (bug,) - else: - purl = urlsplit(bug) - qs = parse_qs(purl.query) - # case 2: long Gentoo bugzilla URL to shorten - if ( - purl.netloc == "bugs.gentoo.org" - and purl.path == "/show_bug.cgi" - and tuple(qs.keys()) == ("id",) - ): - bug = urlunsplit( - ("https", purl.netloc, qs["id"][-1], "", purl.fragment) - ) - # case 3: bug tracker w/ http -> https - elif purl.scheme == "http" and purl.netloc in self.https_bugtrackers: - bug = urlunsplit(("https",) + purl[1:]) - commit_footer += "\n%s: %s" % (tag, bug) - - # Use new footer only for git (see bug #438364). - if self.vcs_settings.vcs in ["git"]: - commit_footer += "\nPackage-Manager: Portage-%s, Repoman-%s" % ( - portage.VERSION, - VERSION, - ) - if report_options: - commit_footer += "\nRepoMan-Options: " + " ".join(report_options) - if self.repo_settings.sign_manifests: - commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key,) - else: - unameout = platform.system() + " " - if platform.system() in ["Darwin", "SunOS"]: - unameout += platform.processor() - else: - unameout += platform.machine() - commit_footer += "\n(Portage version: %s/%s/%s" % ( - portage_version, - self.vcs_settings.vcs, - unameout, - ) - if report_options: - commit_footer += ", RepoMan options: " + " ".join(report_options) - if self.repo_settings.sign_manifests: - commit_footer += ", signed Manifest commit with key %s" % (gpg_key,) - else: - commit_footer += ", unsigned Manifest commit" - commit_footer += ")" - - if signoff: - commit_footer += "\nSigned-off-by: %s" % (signoff,) - - return commit_footer - - def changelogs( - self, - myupdates, - mymanifests, - myremoved, - mychanged, - myautoadd, - mynew, - changelog_msg, - ): - broken_changelog_manifests = [] - if self.options.echangelog in ("y", "force"): - logging.info("checking for unmodified ChangeLog files") - committer_name = utilities.get_committer_name(env=self.repoman_settings) - for x in sorted( - vcs_files_to_cps( - chain(myupdates, mymanifests, myremoved), - self.repo_settings.repodir, - self.scanner.repolevel, - self.scanner.reposplit, - self.scanner.categories, - ) - ): - catdir, pkgdir = x.split("/") - checkdir = self.repo_settings.repodir + "/" + x - checkdir_relative = "" - if self.scanner.repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if self.scanner.repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - changelog_modified = changelog_path in self.scanner.changed.changelogs - if changelog_modified and self.options.echangelog != "force": - continue - - # get changes for this package - cdrlen = len(checkdir_relative) - check_relative = lambda e: e.startswith(checkdir_relative) - split_relative = lambda e: e[cdrlen:] - clnew = list(map(split_relative, filter(check_relative, mynew))) - clremoved = list(map(split_relative, filter(check_relative, myremoved))) - clchanged = list(map(split_relative, filter(check_relative, mychanged))) - - # Skip ChangeLog generation if only the Manifest was modified, - # as discussed in bug #398009. - nontrivial_cl_files = set() - nontrivial_cl_files.update(clnew, clremoved, clchanged) - nontrivial_cl_files.difference_update(["Manifest"]) - if not nontrivial_cl_files and self.options.echangelog != "force": - continue - - new_changelog = utilities.UpdateChangeLog( - checkdir_relative, - committer_name, - changelog_msg, - os.path.join(self.repo_settings.repodir, "skel.ChangeLog"), - catdir, - pkgdir, - new=clnew, - removed=clremoved, - changed=clchanged, - pretend=self.options.pretend, - ) - if new_changelog is None: - writemsg_level( - "!!! Updating the ChangeLog failed\n", - level=logging.ERROR, - noiselevel=-1, - ) - sys.exit(1) - - # if the ChangeLog was just created, add it to vcs - if new_changelog: - myautoadd.append(changelog_path) - # myautoadd is appended to myupdates below - else: - myupdates.append(changelog_path) - - if self.options.ask and not self.options.pretend: - # regenerate Manifest for modified ChangeLog (bug #420735) - self.repoman_settings["O"] = checkdir - digestgen( - mysettings=self.repoman_settings, - myportdb=self.repo_settings.portdb, - ) - else: - broken_changelog_manifests.append(x) - - if myautoadd: - print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") - self.vcs_settings.changes.add_items(myautoadd) - myupdates += myautoadd - return myupdates, broken_changelog_manifests - - def add_manifest(self, mymanifests, myheaders, myupdates, myremoved, commitmessage): - myfiles = mymanifests[:] - # If there are no header (SVN/CVS keywords) changes in - # the files, this Manifest commit must include the - # other (yet uncommitted) files. - if not myheaders: - myfiles += myupdates - myfiles += myremoved - myfiles.sort() - - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - with open(commitmessagefile, "wb") as mymsg: - mymsg.write(_unicode_encode(commitmessage)) - - retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile) - # cleanup the commit message before possibly exiting - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, - noiselevel=-1, - ) - sys.exit(retval) - - def priming_commit(self, myupdates, myremoved, commitmessage): - myfiles = myupdates + myremoved - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - with open(commitmessagefile, "wb") as mymsg: - mymsg.write(_unicode_encode(commitmessage)) - - separator = "-" * 78 - - print() - print(green("Using commit message:")) - print(green(separator)) - print(commitmessage) - print(green(separator)) - print() - - # Having a leading ./ prefix on file paths can trigger a bug in - # the cvs server when committing files to multiple directories, - # so strip the prefix. - myfiles = [f.lstrip("./") for f in myfiles] - - retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile) - # cleanup the commit message before possibly exiting - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, - noiselevel=-1, - ) - sys.exit(retval) - - def sign_manifest(self, myupdates, myremoved, mymanifests): - try: - for x in sorted( - vcs_files_to_cps( - chain(myupdates, myremoved, mymanifests), - self.scanner.repo_settings.repodir, - self.scanner.repolevel, - self.scanner.reposplit, - self.scanner.categories, - ) - ): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - manifest_path = os.path.join(self.repoman_settings["O"], "Manifest") - if not need_signature(manifest_path): - continue - gpgsign(manifest_path, self.repoman_settings, self.options) - except portage.exception.PortageException as e: - portage.writemsg("!!! %s\n" % str(e)) - portage.writemsg("!!! Disabled FEATURES='sign'\n") - self.repo_settings.sign_manifests = False - - def msg_prefix(self): - prefix = "" - if self.scanner.repolevel > 1: - prefix = "/".join(self.scanner.reposplit[1:]) + ": " - return prefix - - def get_new_commit_message(self, qa_output, old_message=None): - msg_prefix = old_message or self.msg_prefix() - try: - editor = os.environ.get("EDITOR") - if editor and utilities.editor_is_executable(editor): - commitmessage = utilities.get_commit_message_with_editor( - editor, message=qa_output, prefix=msg_prefix - ) - else: - print( - "EDITOR is unset or invalid. Please set EDITOR to your preferred editor." - ) - print(bad("* no EDITOR found for commit message, aborting commit.")) - sys.exit(1) - except KeyboardInterrupt: - logging.fatal("Interrupted; exiting...") - sys.exit(1) - if ( - not commitmessage - or not commitmessage.strip() - or commitmessage.strip() == msg_prefix - ): - print("* no commit message? aborting commit.") - sys.exit(1) - return commitmessage - - @staticmethod - def verify_commit_message(msg): - """ - Check whether the message roughly complies with GLEP66. Returns - (True, None) if it does, (False, <explanation>) if it does not. - """ - - problems = [] - paras = msg.strip().split("\n\n") - summary = paras.pop(0) - - if ":" not in summary: - problems.append( - 'summary line must start with a logical unit name, e.g. "cat/pkg:"' - ) - if "\n" in summary.strip(): - problems.append( - "commit message must start with a *single* line of summary, followed by empty line" - ) - # accept 69 overall or unit+50, in case of very long package names - elif len(summary.strip()) > 69 and len(summary.split(":", 1)[-1]) > 50: - problems.append("summary line is too long (max 69 characters)") - - multiple_footers = False - gentoo_bug_used = False - bug_closes_without_url = False - body_too_long = False - - footer_re = re.compile(r"^[\w-]+:") - - found_footer = False - for p in paras: - lines = p.splitlines() - # if all lines look like footer material, we assume it's footer - # else, we assume it's body text - if all(footer_re.match(l) for l in lines if l.strip()): - # multiple footer-like blocks? - if found_footer: - multiple_footers = True - found_footer = True - for l in lines: - if l.startswith("Gentoo-Bug"): - gentoo_bug_used = True - elif l.startswith("Bug:") or l.startswith("Closes:"): - if "http://" not in l and "https://" not in l: - bug_closes_without_url = True - else: - for l in lines: - # we recommend wrapping at 72 but accept up to 80; - # do not complain if we have single word (usually - # it means very long URL) - if len(l.strip()) > 80 and len(l.split()) > 1: - body_too_long = True - - if multiple_footers: - problems.append( - "multiple footer-style blocks found, please combine them into one" - ) - if gentoo_bug_used: - problems.append( - "please replace Gentoo-Bug with GLEP 66-compliant Bug/Closes" - ) - if bug_closes_without_url: - problems.append("Bug/Closes footers require full URL") - if body_too_long: - problems.append("body lines should be wrapped at 72 (max 80) characters") - - if problems: - return (False, "\n".join("- %s" % x for x in problems)) - return (True, None) diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py deleted file mode 100644 index 019c7381b25..00000000000 --- a/repoman/lib/repoman/argparser.py +++ /dev/null @@ -1,388 +0,0 @@ -# repoman: Argument parser -# Copyright 2007-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains functions used in Repoman to parse CLI arguments.""" - -import argparse -import logging - -# import our initialized portage instance -from repoman._portage import portage - -from portage import _unicode_decode -from portage import util - - -def parse_args(argv, repoman_default_opts): - """Use a customized optionParser to parse command line arguments for repoman - Args: - argv - a sequence of command line arguments - Returns: - (opts, args), just like a call to parser.parse_args() - """ - - argv = portage._decode_argv(argv) - - modes = { - "commit": "Run a scan then commit changes", - "ci": "Run a scan then commit changes", - "fix": "Fix simple QA issues (stray digests, missing digests)", - "full": "Scan directory tree and print all issues (not a summary)", - "help": "Show this screen", - "manifest": "Generate a Manifest (fetches files if necessary)", - "manifest-check": "Check Manifests for missing or incorrect digests", - "scan": "Scan directory tree for QA issues", - } - - output_choices = { - "default": "The normal output format", - "column": "Columnar output suitable for use with grep", - } - - mode_keys = list(modes) - mode_keys.sort() - - output_keys = sorted(output_choices) - - parser = argparse.ArgumentParser( - usage="repoman [options] [mode]", - description="Modes: %s" % " | ".join(mode_keys), - epilog="For more help consult the man page.", - ) - - parser.add_argument( - "-a", - "--ask", - dest="ask", - action="store_true", - default=False, - help="Request a confirmation before commiting", - ) - - parser.add_argument( - "-b", - "--bug", - dest="bug", - action="append", - metavar="<BUG-NO|BUG-URL>", - default=[], - help=( - "Mention a Gentoo or upstream bug in the commit footer; " - "takes either Gentoo bug number or full bug URL" - ), - ) - - parser.add_argument( - "-c", - "--closes", - dest="closes", - action="append", - metavar="<PR-NO|PR-URL>", - default=[], - help=( - "Adds a Closes footer to close GitHub pull request (or compatible); " - "takes either GitHub PR number or full PR URL" - ), - ) - - parser.add_argument( - "-m", - "--commitmsg", - dest="commitmsg", - help="specify a commit message on the command line", - ) - - parser.add_argument( - "-M", - "--commitmsgfile", - dest="commitmsgfile", - help="specify a path to a file that contains a commit message", - ) - - parser.add_argument( - "--digest", - choices=("y", "n"), - metavar="<y|n>", - help="Automatically update Manifest digests for modified files", - ) - - parser.add_argument( - "-p", - "--pretend", - dest="pretend", - default=False, - action="store_true", - help="don't commit or fix anything; just show what would be done", - ) - - parser.add_argument( - "-q", - "--quiet", - dest="quiet", - action="count", - default=0, - help="do not print unnecessary messages", - ) - - parser.add_argument( - "--echangelog", - choices=("y", "n", "force"), - metavar="<y|n|force>", - help=( - "for commit mode, call echangelog if ChangeLog is unmodified (or " - "regardless of modification if 'force' is specified)" - ), - ) - - parser.add_argument( - "--experimental-inherit", - choices=("y", "n"), - metavar="<y|n>", - default="n", - help=( - "Enable experimental inherit.missing checks which may misbehave" - " when the internal eclass database becomes outdated" - ), - ) - - parser.add_argument( - "--experimental-repository-modules", - choices=("y", "n"), - metavar="<y|n>", - default="n", - help="Enable experimental repository modules", - ) - - parser.add_argument( - "-f", - "--force", - dest="force", - action="store_true", - default=False, - help="Commit with QA violations", - ) - - parser.add_argument( - "-S", - "--straight-to-stable", - dest="straight_to_stable", - default=False, - action="store_true", - help="Allow committing straight to stable", - ) - - parser.add_argument( - "--vcs", dest="vcs", help="Force using specific VCS instead of autodetection" - ) - - parser.add_argument( - "-v", - "--verbose", - dest="verbosity", - action="count", - help="be very verbose in output", - default=0, - ) - - parser.add_argument( - "-V", "--version", dest="version", action="store_true", help="show version info" - ) - - parser.add_argument( - "-x", - "--xmlparse", - dest="xml_parse", - action="store_true", - default=False, - help="forces the metadata.xml parse check to be carried out", - ) - - parser.add_argument( - "--if-modified", - choices=("y", "n"), - default="n", - metavar="<y|n>", - help="only check packages that have uncommitted modifications", - ) - - parser.add_argument( - "-i", - "--ignore-arches", - dest="ignore_arches", - action="store_true", - default=False, - help="ignore arch-specific failures (where arch != host)", - ) - - parser.add_argument( - "--ignore-default-opts", - action="store_true", - help="do not use the REPOMAN_DEFAULT_OPTS environment variable", - ) - - parser.add_argument( - "-I", - "--ignore-masked", - dest="ignore_masked", - action="store_true", - default=False, - help="ignore masked packages (not allowed with commit mode)", - ) - - parser.add_argument( - "--include-arches", - dest="include_arches", - metavar="ARCHES", - action="append", - help=( - "A space separated list of arches used to " - "filter the selection of profiles for dependency checks" - ), - ) - - parser.add_argument( - "--include-profiles", - dest="include_profiles", - metavar="PROFILES", - action="append", - help=( - "A space separated list of profiles used to " - "define the selection of profiles for dependency checks" - ), - ) - - parser.add_argument( - "-d", - "--include-dev", - dest="include_dev", - action="store_true", - default=False, - help="include dev profiles in dependency checks", - ) - - parser.add_argument( - "-e", - "--include-exp-profiles", - choices=("y", "n"), - metavar="<y|n>", - default=False, - help="include exp profiles in dependency checks", - ) - - parser.add_argument( - "--unmatched-removal", - dest="unmatched_removal", - action="store_true", - default=False, - help=( - "enable strict checking of package.mask and package.unmask files" - " for unmatched removal atoms" - ), - ) - - parser.add_argument( - "--without-mask", - dest="without_mask", - action="store_true", - default=False, - help=( - "behave as if no package.mask entries exist" - " (not allowed with commit mode)" - ), - ) - - parser.add_argument( - "--output-style", - dest="output_style", - choices=output_keys, - help="select output type", - default="default", - ) - - parser.add_argument( - "-j", - "--jobs", - dest="jobs", - action="store", - type=int, - default=1, - help="Specifies the number of jobs (processes) to run simultaneously.", - ) - - parser.add_argument( - "-l", - "--load-average", - dest="load_average", - action="store", - type=float, - default=None, - help="Specifies that no new jobs (processes) should be started if there are others " - "jobs running and the load average is at least load (a floating-point number).", - ) - - parser.add_argument( - "--mode", - dest="mode", - choices=mode_keys, - help="specify which mode repoman will run in (default=full)", - ) - - # Modes help is included earlier, in the parser description. - parser.add_argument( - "mode_positional", - nargs="?", - metavar="mode", - choices=mode_keys, - help=argparse.SUPPRESS, - ) - - opts = parser.parse_args(argv[1:]) - - if not opts.ignore_default_opts: - default_opts = util.shlex_split(repoman_default_opts) - if default_opts: - opts = parser.parse_args(default_opts + argv[1:]) - - args = [] - if opts.mode is not None: - args.append(opts.mode) - if opts.mode_positional is not None: - args.append(opts.mode_positional) - - if len(set(args)) > 1: - parser.error("multiple modes specified: %s" % " ".join(args)) - - opts.mode = args[0] if args else None - - if opts.mode == "help": - parser.print_help() - parser.exit() - - if not opts.mode: - opts.mode = "full" - - if opts.mode == "ci": - opts.mode = "commit" # backwards compat shortcut - - # Use verbosity and quiet options to appropriately fiddle with the loglevel - for val in range(opts.verbosity): - logger = logging.getLogger() - logger.setLevel(logger.getEffectiveLevel() - 10) - - for val in range(opts.quiet): - logger = logging.getLogger() - logger.setLevel(logger.getEffectiveLevel() + 10) - - if opts.mode == "commit" and opts.commitmsg: - opts.commitmsg = _unicode_decode(opts.commitmsg) - - if opts.mode == "commit" and not (opts.force or opts.pretend): - if opts.ignore_masked: - opts.ignore_masked = False - logging.warn("Commit mode automatically disables --ignore-masked") - if opts.without_mask: - opts.without_mask = False - logging.warn("Commit mode automatically disables --without-mask") - - return (opts, args) diff --git a/repoman/lib/repoman/check_missingslot.py b/repoman/lib/repoman/check_missingslot.py deleted file mode 100644 index 4d2d7676b3a..00000000000 --- a/repoman/lib/repoman/check_missingslot.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: missing slot check -# Copyright 2014 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains the check used to find missing slot values -in dependencies.""" - -from portage.eapi import eapi_has_slot_operator - - -def check_missingslot(atom, mytype, eapi, portdb, qatracker, relative_path, my_aux): - # If no slot or slot operator is specified in RDEP... - if ( - not atom.blocker - and not atom.slot - and not atom.slot_operator - and mytype == "RDEPEND" - and eapi_has_slot_operator(eapi) - ): - # Check whether it doesn't match more than one. - atom_matches = portdb.xmatch("match-all", atom) - dep_slots = frozenset( - portdb.aux_get(cpv, ["SLOT"])[0].split("/")[0] for cpv in atom_matches - ) - - if len(dep_slots) > 1: - # See if it is a DEPEND as well. It's a very simple & dumb - # check but should suffice for catching it. - depend = my_aux["DEPEND"].split() - if atom not in depend: - return - - qatracker.add_error( - "dependency.missingslot", - relative_path - + ": %s: '%s' matches more than one slot, please specify an explicit slot and/or use the := or :* slot operator" - % (mytype, atom), - ) diff --git a/repoman/lib/repoman/checks/__init__.py b/repoman/lib/repoman/checks/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/checks/__init__.py +++ /dev/null diff --git a/repoman/lib/repoman/config.py b/repoman/lib/repoman/config.py deleted file mode 100644 index 621143de4f6..00000000000 --- a/repoman/lib/repoman/config.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding:utf-8 -*- - -import copy -import itertools -import json -import os -import stat - -try: - import yaml -except ImportError: - yaml = None - - -class ConfigError(Exception): - """Raised when a config file fails to load""" - - pass - - -def merge_config(base, head): - """ - Merge two JSON or YAML documents into a single object. Arrays are - merged by extension. If dissimilar types are encountered, then the - head value overwrites the base value. - """ - - if isinstance(head, dict): - if not isinstance(base, dict): - return copy.deepcopy(head) - - result = {} - for k in itertools.chain(head, base): - try: - result[k] = merge_config(base[k], head[k]) - except KeyError: - try: - result[k] = copy.deepcopy(head[k]) - except KeyError: - result[k] = copy.deepcopy(base[k]) - - elif isinstance(head, list): - result = [] - if not isinstance(base, list): - result.extend(copy.deepcopy(x) for x in head) - else: - if any(isinstance(x, (dict, list)) for x in itertools.chain(head, base)): - # merge items with identical indexes - for x, y in zip(base, head): - if isinstance(x, (dict, list)): - result.append(merge_config(x, y)) - else: - # head overwrites base (preserving index) - result.append(copy.deepcopy(y)) - # copy remaining items from the longer list - if len(base) != len(head): - if len(base) > len(head): - result.extend(copy.deepcopy(x) for x in base[len(head) :]) - else: - result.extend(copy.deepcopy(x) for x in head[len(base) :]) - else: - result.extend(copy.deepcopy(x) for x in base) - result.extend(copy.deepcopy(x) for x in head) - - else: - result = copy.deepcopy(head) - - return result - - -def _yaml_load(filename): - """ - Load filename as YAML and return a dict. Raise ConfigError if - it fails to load. - """ - if yaml is None: - raise ImportError("Please install pyyaml in order to read yaml files") - - with open(filename, "rt") as f: - try: - return yaml.safe_load(f) - except yaml.parser.ParserError as e: - raise ConfigError("{}: {}".format(filename, e)) - - -def _json_load(filename): - """ - Load filename as JSON and return a dict. Raise ConfigError if - it fails to load. - """ - with open(filename, "rt") as f: - try: - return json.load(f) # nosec - except ValueError as e: - raise ConfigError("{}: {}".format(filename, e)) - - -def iter_files(files_dirs): - """ - Iterate over nested file paths in lexical order. - """ - stack = list(reversed(files_dirs)) - while stack: - location = stack.pop() - try: - st = os.stat(location) - except FileNotFoundError: - continue - - if stat.S_ISDIR(st.st_mode): - stack.extend( - os.path.join(location, x) - for x in sorted(os.listdir(location), reverse=True) - ) - - elif stat.S_ISREG(st.st_mode): - yield location - - -def load_config(conf_dirs, file_extensions=None, valid_versions=None): - """ - Load JSON and/or YAML files from a directories, and merge them together - into a single object. - - @param conf_dirs: ordered iterable of directories to load the config from - @param file_extensions: Optional list of file extension types to load - @param valid_versions: list of compatible file versions allowed - @returns: the stacked config - """ - - result = {} - for filename in iter_files(conf_dirs): - if file_extensions is not None and not filename.endswith(file_extensions): - continue - - loaders = [] - extension = filename.rsplit(".", 1)[1] - if extension in ["json"]: - loaders.append(_json_load) - elif extension in ["yml", "yaml"]: - loaders.append(_yaml_load) - - config = None - exception = None - for loader in loaders: - try: - config = loader(filename) or {} - except ConfigError as e: - exception = e - else: - break - - if config is None: - print("Repoman.config.load_config(), Error loading file: %s" % filename) - print(" Aborting...") - raise exception - - if config: - if config["version"] not in valid_versions: - raise ConfigError( - "Invalid file version: %s in: %s\nPlease upgrade to " - ">=app-portage/repoman-%s, current valid API versions: %s" - % ( - config["version"], - filename, - config["repoman_version"], - valid_versions, - ) - ) - result = merge_config(result, config) - - return result diff --git a/repoman/lib/repoman/copyrights.py b/repoman/lib/repoman/copyrights.py deleted file mode 100644 index 199f5efbfe9..00000000000 --- a/repoman/lib/repoman/copyrights.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding:utf-8 -*- - - -import difflib -import io -import re -from tempfile import mkstemp - -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -from portage import os -from portage import shutil -from portage import util - - -_copyright_re1 = re.compile( - rb"^(# Copyright \d\d\d\d)-\d\d\d\d( Gentoo (Foundation|Authors))\b" -) -_copyright_re2 = re.compile( - rb"^(# Copyright )(\d\d\d\d)( Gentoo (Foundation|Authors))\b" -) - - -class _copyright_repl: - __slots__ = ("year",) - - def __init__(self, year): - self.year = year - - def __call__(self, matchobj): - if matchobj.group(2) == self.year: - return matchobj.group(0) - else: - return ( - matchobj.group(1) - + matchobj.group(2) - + b"-" - + self.year - + b" Gentoo Authors" - ) - - -def update_copyright_year(year, line): - """ - These two regexes are taken from echangelog - update_copyright(), except that we don't hardcode - 1999 here (in order to be more generic). - """ - is_bytes = isinstance(line, bytes) - if is_bytes: - if not line.startswith(b"# Copyright "): - return line - else: - if not line.startswith("# Copyright "): - return line - - year = _unicode_encode(year) - line = _unicode_encode(line) - - line = _copyright_re1.sub(rb"\1-" + year + b" Gentoo Authors", line) - line = _copyright_re2.sub(_copyright_repl(year), line) - if not is_bytes: - line = _unicode_decode(line) - return line - - -def update_copyright(fn_path, year, pretend=False): - """ - Check file for a Copyright statement, and update its year. The - patterns used for replacing copyrights are taken from echangelog. - Only the first lines of each file that start with a hash ('#') are - considered, until a line is found that doesn't start with a hash. - Files are read and written in binary mode, so that this function - will work correctly with files encoded in any character set, as - long as the copyright statements consist of plain ASCII. - - @param fn_path: file path - @type str - @param year: current year - @type str - @param pretend: pretend mode - @type bool - @rtype: bool - @return: True if copyright update was needed, False otherwise - """ - - try: - fn_hdl = io.open( - _unicode_encode(fn_path, encoding=_encodings["fs"], errors="strict"), - mode="rb", - ) - except EnvironmentError: - return - - orig_header = [] - new_header = [] - - for line in fn_hdl: - line_strip = line.strip() - orig_header.append(line) - if not line_strip or line_strip[:1] != b"#": - new_header.append(line) - break - - line = update_copyright_year(year, line) - new_header.append(line) - - difflines = 0 - for diffline in difflib.unified_diff( - [_unicode_decode(diffline) for diffline in orig_header], - [_unicode_decode(diffline) for diffline in new_header], - fromfile=fn_path, - tofile=fn_path, - n=0, - ): - util.writemsg_stdout(diffline, noiselevel=-1) - difflines += 1 - util.writemsg_stdout("\n", noiselevel=-1) - - # unified diff has three lines to start with - if difflines > 3 and not pretend: - # write new file with changed header - f, fnnew_path = mkstemp() - f = io.open(f, mode="wb") - for line in new_header: - f.write(line) - for line in fn_hdl: - f.write(line) - f.close() - try: - fn_stat = os.stat(fn_path) - except OSError: - fn_stat = None - - shutil.move(fnnew_path, fn_path) - - if fn_stat is None: - util.apply_permissions(fn_path, mode=0o644) - else: - util.apply_stat_permissions(fn_path, fn_stat) - fn_hdl.close() - return difflines > 3 diff --git a/repoman/lib/repoman/errors.py b/repoman/lib/repoman/errors.py deleted file mode 100644 index c3ee92d37f6..00000000000 --- a/repoman/lib/repoman/errors.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding:utf-8 -*- - -import sys - - -def warn(txt): - print("repoman: " + txt) - - -def err(txt): - warn(txt) - sys.exit(1) - - -def caterror(catdir, repodir): - err( - "%s is not an official category." - " Skipping QA checks in this directory.\n" - "Please ensure that you add %s to %s/profiles/categories\n" - "if it is a new category." % (catdir, catdir, repodir) - ) diff --git a/repoman/lib/repoman/gpg.py b/repoman/lib/repoman/gpg.py deleted file mode 100644 index 10e884e52bb..00000000000 --- a/repoman/lib/repoman/gpg.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding:utf-8 -*- - -import errno -import logging -import subprocess - -import portage -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.exception import MissingParameter - - -# Setup the GPG commands -def gpgsign(filename, repoman_settings, options): - gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND") - if gpgcmd in [None, ""]: - raise MissingParameter( - "PORTAGE_GPG_SIGNING_COMMAND is unset!" " Is make.globals missing?" - ) - if "${PORTAGE_GPG_KEY}" in gpgcmd and "PORTAGE_GPG_KEY" not in repoman_settings: - raise MissingParameter("PORTAGE_GPG_KEY is unset!") - if "${PORTAGE_GPG_DIR}" in gpgcmd: - if "PORTAGE_GPG_DIR" not in repoman_settings: - repoman_settings["PORTAGE_GPG_DIR"] = os.path.expanduser("~/.gnupg") - logging.info( - "Automatically setting PORTAGE_GPG_DIR to '%s'" - % repoman_settings["PORTAGE_GPG_DIR"] - ) - else: - repoman_settings["PORTAGE_GPG_DIR"] = os.path.expanduser( - repoman_settings["PORTAGE_GPG_DIR"] - ) - if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK): - raise portage.exception.InvalidLocation( - "Unable to access directory: PORTAGE_GPG_DIR='%s'" - % repoman_settings["PORTAGE_GPG_DIR"] - ) - gpgvars = {"FILE": filename} - for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): - v = repoman_settings.get(k) - if v is not None: - gpgvars[k] = v - gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) - if options.pretend: - print("(" + gpgcmd + ")") - else: - # Encode unicode manually for bug #310789. - gpgcmd = portage.util.shlex_split(gpgcmd) - - gpgcmd = [ - _unicode_encode(arg, encoding=_encodings["fs"], errors="strict") - for arg in gpgcmd - ] - rValue = subprocess.call(gpgcmd) - if rValue == os.EX_OK: - os.rename(filename + ".asc", filename) - else: - raise portage.exception.PortageException( - "!!! gpg exited with '" + str(rValue) + "' status" - ) - - -def need_signature(filename): - try: - with open( - _unicode_encode(filename, encoding=_encodings["fs"], errors="strict"), "rb" - ) as f: - return b"BEGIN PGP SIGNED MESSAGE" not in f.readline() - except IOError as e: - if e.errno in (errno.ENOENT, errno.ESTALE): - return False - raise diff --git a/repoman/lib/repoman/main.py b/repoman/lib/repoman/main.py deleted file mode 100755 index 1cfe86e8d55..00000000000 --- a/repoman/lib/repoman/main.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# Copyright 1999-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import collections -import io -import logging -import sys - -# import our centrally initialized portage instance -from repoman._portage import portage - -portage._internal_caller = True -portage._disable_legacy_globals() - - -from portage import os -import portage.checksum -import portage.const -import portage.repository.config -from portage.output import create_color_func, nocolor -from portage.output import ConsoleStyleFile, StyleWriter -from portage.util import formatter -from portage.util.futures.extendedfutures import ( - ExtendedFuture, -) - -# pylint: disable=ungrouped-imports -from repoman.actions import Actions -from repoman.argparser import parse_args -from repoman.qa_data import QAData -from repoman.qa_data import format_qa_output, format_qa_output_column -from repoman.repos import RepoSettings -from repoman.scanner import Scanner -from repoman import utilities -from repoman.modules.vcs.settings import VCSSettings -from repoman import VERSION - - -bad = create_color_func("BAD") - -# A sane umask is needed for files that portage creates. -os.umask(0o22) - -LOGLEVEL = logging.WARNING -portage.util.initialize_logger(LOGLEVEL) - -VALID_VERSIONS = [ - 1, -] - -_repoman_main_vars = collections.namedtuple( - "_repoman_main_vars", - ( - "can_force", - "exitcode", - "options", - "qadata", - "repo_settings", - "scanner", - "vcs_settings", - ), -) - - -def repoman_main(argv): - repoman_vars = _repoman_init(argv) - if repoman_vars.exitcode is not None: - return repoman_vars.exitcode - result = _repoman_scan(*repoman_vars) - return _handle_result(*repoman_vars, result) - - -def _repoman_init(argv): - config_root = os.environ.get("PORTAGE_CONFIGROOT") - repoman_settings = portage.config(config_root=config_root, local_config=False) - repoman_settings.valid_versions = VALID_VERSIONS - - if ( - repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") - or repoman_settings.get("TERM") == "dumb" - or not sys.stdout.isatty() - ): - nocolor() - - options, arguments = parse_args( - argv, repoman_settings.get("REPOMAN_DEFAULT_OPTS", "") - ) - - if options.version: - print("Repoman", VERSION, "(portage-%s)" % portage.VERSION) - return _repoman_main_vars(None, 0, None, None, None, None, None) - - logger = logging.getLogger() - - if options.verbosity > 0: - logger.setLevel(LOGLEVEL - 10 * options.verbosity) - else: - logger.setLevel(LOGLEVEL) - - # Set this to False when an extraordinary issue (generally - # something other than a QA issue) makes it impossible to - # commit (like if Manifest generation fails). - can_force = ExtendedFuture(True) - repo_settings, vcs_settings, scanner, qadata = _create_scanner( - options, can_force, config_root, repoman_settings - ) - return _repoman_main_vars( - can_force, None, options, qadata, repo_settings, scanner, vcs_settings - ) - - -def _create_scanner(options, can_force, config_root, repoman_settings): - - portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) - if portdir is None: - return (None, None, None, None) - - myreporoot = os.path.basename(portdir_overlay) - myreporoot += mydir[len(portdir_overlay) :] - - # avoid a circular parameter repo_settings - vcs_settings = VCSSettings(options, repoman_settings) - qadata = QAData() - - logging.debug("repoman_main: RepoSettings init") - repo_settings = RepoSettings( - config_root, - portdir, - portdir_overlay, - repoman_settings, - vcs_settings, - options, - qadata, - ) - repoman_settings = repo_settings.repoman_settings - repoman_settings.valid_versions = VALID_VERSIONS - - # Now set repo_settings - vcs_settings.repo_settings = repo_settings - # set QATracker qacats, qawarnings - vcs_settings.qatracker.qacats = repo_settings.qadata.qacats - vcs_settings.qatracker.qawarnings = repo_settings.qadata.qawarnings - logging.debug("repoman_main: vcs_settings done") - logging.debug("repoman_main: qadata: %s", repo_settings.qadata) - - if "digest" in repoman_settings.features and options.digest != "n": - options.digest = "y" - - logging.debug("vcs: %s" % (vcs_settings.vcs,)) - logging.debug("repo config: %s" % (repo_settings.repo_config,)) - logging.debug("options: %s" % (options,)) - - # It's confusing if these warnings are displayed without the user - # being told which profile they come from, so disable them. - env = os.environ.copy() - env["FEATURES"] = env.get("FEATURES", "") + " -unknown-features-warn" - - # Perform the main checks - scanner = Scanner( - repo_settings, myreporoot, config_root, options, vcs_settings, mydir, env - ) - return repo_settings, vcs_settings, scanner, qadata - - -def _repoman_scan( - can_force, exitcode, options, qadata, repo_settings, scanner, vcs_settings -): - scanner.scan_pkgs(can_force) - - if options.if_modified == "y" and len(scanner.effective_scanlist) < 1: - logging.warning( - "--if-modified is enabled, but no modified packages were found!" - ) - - result = { - # fail will be true if we have failed in at least one non-warning category - "fail": 0, - # warn will be true if we tripped any warnings - "warn": 0, - # full will be true if we should print a "repoman full" informational message - "full": options.mode != "full", - } - - for x in qadata.qacats: - if x not in vcs_settings.qatracker.fails: - continue - result["warn"] = 1 - if x not in qadata.qawarnings: - result["fail"] = 1 - - if result["fail"] or ( - result["warn"] and not (options.quiet or options.mode == "scan") - ): - result["full"] = 0 - - return result - - -def _handle_result( - can_force, exitcode, options, qadata, repo_settings, scanner, vcs_settings, result -): - commitmessage = None - if options.commitmsg: - commitmessage = options.commitmsg - elif options.commitmsgfile: - # we don't need the actual text of the commit message here - # the filename will do for the next code block - commitmessage = options.commitmsgfile - - # Save QA output so that it can be conveniently displayed - # in $EDITOR while the user creates a commit message. - # Otherwise, the user would not be able to see this output - # once the editor has taken over the screen. - qa_output = io.StringIO() - style_file = ConsoleStyleFile(sys.stdout) - if options.mode == "commit" and (not commitmessage or not commitmessage.strip()): - style_file.write_listener = qa_output - console_writer = StyleWriter(file=style_file, maxcol=9999) - console_writer.style_listener = style_file.new_styles - - f = formatter.AbstractFormatter(console_writer) - - format_outputs = {"column": format_qa_output_column, "default": format_qa_output} - - format_output = format_outputs.get(options.output_style, format_outputs["default"]) - format_output( - f, - vcs_settings.qatracker.fails, - result["full"], - result["fail"], - options, - qadata.qawarnings, - ) - - style_file.flush() - del console_writer, f, style_file - - # early out for manifest generation - if options.mode == "manifest": - return 1 if result["fail"] else 0 - - qa_output = qa_output.getvalue() - qa_output = qa_output.splitlines(True) - - # output the results - actions = Actions(repo_settings, options, scanner, vcs_settings) - if actions.inform(can_force.get(), result): - # perform any other actions - actions.perform(qa_output) - elif result["fail"]: - return 1 - - return 0 diff --git a/repoman/lib/repoman/metadata.py b/repoman/lib/repoman/metadata.py deleted file mode 100644 index 5dc44b11947..00000000000 --- a/repoman/lib/repoman/metadata.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding:utf-8 -*- - -import errno -import logging -import time - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.output import green -from portage.package.ebuild.fetch import fetch - - -# Note: This URI is hardcoded in all metadata.xml files. We can't -# change it without updating all the xml files in the tree. -metadata_dtd_uri = "https://www.gentoo.org/dtd/metadata.dtd" -metadata_xsd_uri = "https://www.gentoo.org/xml-schema/metadata.xsd" -# force refetch if the local copy creation time is older than this -metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days - - -def fetch_metadata_xsd(metadata_xsd, repoman_settings): - """ - Fetch metadata.xsd if it doesn't exist or the ctime is older than - metadata_xsd_ctime_interval. - @rtype: bool - @return: True if successful, otherwise False - """ - - must_fetch = True - metadata_xsd_st = None - current_time = int(time.time()) - try: - metadata_xsd_st = os.stat(metadata_xsd) - except EnvironmentError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - del e - else: - # Trigger fetch if metadata.xsd mtime is old or clock is wrong. - if abs(current_time - metadata_xsd_st.st_ctime) < metadata_xsd_ctime_interval: - must_fetch = False - - if must_fetch: - print() - print( - "%s the local copy of metadata.xsd " - "needs to be refetched, doing that now" % green("***") - ) - print() - - if not fetch([metadata_xsd_uri], repoman_settings, force=1, try_mirrors=0): - logging.error("failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri) - return False - - try: - portage.util.apply_secpass_permissions( - metadata_xsd, gid=portage.data.portage_gid, mode=0o664, mask=0o2 - ) - except portage.exception.PortageException: - pass - - return True - - -def get_metadata_xsd(repo_settings): - """Locate and or fetch the metadata.xsd file - - @param repo_settings: RepoSettings instance - @returns: path to the metadata.xsd file - """ - metadata_xsd = None - paths = list(repo_settings.repo_config.eclass_db.porttrees) - paths.reverse() - # add the test copy - paths.append("/usr/lib/portage/cnf/") - for path in paths: - path = os.path.join(path, "metadata/xml-schema/metadata.xsd") - if os.path.exists(path): - metadata_xsd = path - break - if metadata_xsd is None: - metadata_xsd = os.path.join( - repo_settings.repoman_settings["DISTDIR"], "metadata.xsd" - ) - - fetch_metadata_xsd(metadata_xsd, repo_settings.repoman_settings) - return metadata_xsd diff --git a/repoman/lib/repoman/modules/__init__.py b/repoman/lib/repoman/modules/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/modules/__init__.py +++ /dev/null diff --git a/repoman/lib/repoman/modules/commit/__init__.py b/repoman/lib/repoman/modules/commit/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/modules/commit/__init__.py +++ /dev/null diff --git a/repoman/lib/repoman/modules/commit/manifest.py b/repoman/lib/repoman/modules/commit/manifest.py deleted file mode 100644 index e282ad40694..00000000000 --- a/repoman/lib/repoman/modules/commit/manifest.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding:utf-8 -*- - -import logging - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.util import writemsg_level - - -class Manifest: - """Creates as well as checks pkg Manifest entries/files""" - - def __init__(self, **kwargs): - """Class init - - @param options: the run time cli options - @param portdb: portdb instance - @param repo_settings: repository settings instance - """ - self.options = kwargs.get("options") - self.portdb = kwargs.get("portdb") - self.repoman_settings = kwargs.get("repo_settings").repoman_settings - self.generated_manifest = False - - def update_manifest(self, checkdir): - """Perform a manifest generation for the pkg - - @param checkdir: the current package directory - @rtype: bool - @return: True if successful, False otherwise - """ - self.generated_manifest = False - failed = False - self.auto_assumed = set() - fetchlist_dict = portage.FetchlistDict( - checkdir, self.repoman_settings, self.portdb - ) - if self.options.mode == "manifest" and self.options.force: - self._discard_dist_digests(checkdir, fetchlist_dict) - self.repoman_settings["O"] = checkdir - try: - self.generated_manifest = digestgen( - mysettings=self.repoman_settings, myportdb=self.portdb - ) - except portage.exception.PermissionDenied as e: - self.generated_manifest = False - writemsg_level( - "!!! Permission denied: '%s'\n" % (e,), - level=logging.ERROR, - noiselevel=-1, - ) - - if not self.generated_manifest: - writemsg_level( - "!!! Unable to generate manifest for '%s'.\n" % (checkdir,), - level=logging.ERROR, - noiselevel=-1, - ) - failed = True - - if self.options.mode == "manifest": - if ( - not failed - and self.options.force - and self.auto_assumed - and "assume-digests" in self.repoman_settings.features - ): - # Show which digests were assumed despite the --force option - # being given. This output will already have been shown by - # digestgen() if assume-digests is not enabled, so only show - # it here if assume-digests is enabled. - pkgs = list(fetchlist_dict) - pkgs.sort() - portage.writemsg_stdout( - " digest.assumed %s" - % portage.output.colorize( - "WARN", str(len(self.auto_assumed)).rjust(18) - ) - + "\n" - ) - for cpv in pkgs: - fetchmap = fetchlist_dict[cpv] - pf = portage.catsplit(cpv)[1] - for distfile in sorted(fetchmap): - if distfile in self.auto_assumed: - portage.writemsg_stdout(" %s::%s\n" % (pf, distfile)) - return not failed - - def _discard_dist_digests(self, checkdir, fetchlist_dict): - """Discard DIST digests for files that exist in DISTDIR - - This method is intended to be called prior to digestgen, only for - manifest mode with the --force option, in order to discard DIST - digests that we intend to update. This is necessary because - digestgen never replaces existing digests, since otherwise it - would be too easy for ebuild developers to accidentally corrupt - existing DIST digests. - - @param checkdir: the directory to generate the Manifest in - @param fetchlist_dict: dictionary of files to fetch and/or include - in the manifest - """ - portage._doebuild_manifest_exempt_depend += 1 - try: - distdir = self.repoman_settings["DISTDIR"] - mf = self.repoman_settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(checkdir)) - ) - mf = mf.load_manifest(checkdir, distdir, fetchlist_dict=fetchlist_dict) - mf.create(requiredDistfiles=None, assumeDistHashesAlways=True) - for distfiles in fetchlist_dict.values(): - for distfile in distfiles: - if os.path.isfile(os.path.join(distdir, distfile)): - mf.fhashdict["DIST"].pop(distfile, None) - else: - self.auto_assumed.add(distfile) - mf.write() - finally: - portage._doebuild_manifest_exempt_depend -= 1 diff --git a/repoman/lib/repoman/modules/commit/repochecks.py b/repoman/lib/repoman/modules/commit/repochecks.py deleted file mode 100644 index 32466021e05..00000000000 --- a/repoman/lib/repoman/modules/commit/repochecks.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding:utf-8 -*- - -from portage.output import red - -from repoman.errors import err - - -def commit_check(repolevel, reposplit): - # Check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting. - # Reason for this is if they're trying to commit in just $FILESDIR/*, - # the Manifest needs updating. - # This check ensures that repoman knows where it is, - # and the manifest recommit is at least possible. - if repolevel not in [1, 2, 3]: - print( - red("***") - + ( - " Commit attempts *must* be from within a vcs checkout," - " category, or package directory." - ) - ) - print( - red("***") - + ( - " Attempting to commit from a packages files directory" - " will be blocked for instance." - ) - ) - print( - red("***") - + ( - " This is intended behaviour," - " to ensure the manifest is recommitted for a package." - ) - ) - print(red("***")) - err( - "Unable to identify level we're commiting from for %s" % "/".join(reposplit) - ) - - -def conflict_check(vcs_settings, options): - if vcs_settings.vcs: - conflicts = vcs_settings.status.detect_conflicts(options) diff --git a/repoman/lib/repoman/modules/linechecks/__init__.py b/repoman/lib/repoman/modules/linechecks/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/modules/linechecks/__init__.py +++ /dev/null diff --git a/repoman/lib/repoman/modules/linechecks/assignment/__init__.py b/repoman/lib/repoman/modules/linechecks/assignment/__init__.py deleted file mode 100644 index df2844cf5b9..00000000000 --- a/repoman/lib/repoman/modules/linechecks/assignment/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Assignment plug-in module for repoman LineChecks. -Performs assignments checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "assignment", - "description": doc, - "provides": { - "assignment-check": { - "name": "assignment", - "sourcefile": "assignment", - "class": "EbuildAssignment", - "description": doc, - }, - "eapi3-check": { - "name": "eapi3assignment", - "sourcefile": "assignment", - "class": "Eapi3EbuildAssignment", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/assignment/assignment.py b/repoman/lib/repoman/modules/linechecks/assignment/assignment.py deleted file mode 100644 index 2fbdef7ae6b..00000000000 --- a/repoman/lib/repoman/modules/linechecks/assignment/assignment.py +++ /dev/null @@ -1,38 +0,0 @@ -import re - -from portage.eapi import eapi_supports_prefix, eapi_has_broot -from repoman.modules.linechecks.base import LineCheck - - -class EbuildAssignment(LineCheck): - """Ensure ebuilds don't assign to readonly variables.""" - - repoman_check_name = "variable.readonly" - read_only_vars = "A|CATEGORY|P|P[VNRF]|PVR|D|WORKDIR|FILESDIR|FEATURES|USE" - readonly_assignment = re.compile(r"^\s*(export\s+)?(%s)=" % read_only_vars) - - def check(self, num, line): - match = self.readonly_assignment.match(line) - e = None - if match is not None: - e = self.errors["READONLY_ASSIGNMENT_ERROR"] - return e - - -class Eapi3EbuildAssignment(EbuildAssignment): - """Ensure ebuilds don't assign to readonly EAPI 3-introduced variables.""" - - read_only_vars = "ED|EPREFIX|EROOT" - readonly_assignment = re.compile(r"\s*(export\s+)?(%s)=" % read_only_vars) - - def check_eapi(self, eapi): - return eapi_supports_prefix(eapi) - - -class Eapi7EbuildAssignment(EbuildAssignment): - """Ensure ebuilds don't assign to readonly EAPI 7-introduced variables.""" - - readonly_assignment = re.compile(r"\s*(export\s+)?BROOT=") - - def check_eapi(self, eapi): - return eapi_has_broot(eapi) diff --git a/repoman/lib/repoman/modules/linechecks/base.py b/repoman/lib/repoman/modules/linechecks/base.py deleted file mode 100644 index 851057940dd..00000000000 --- a/repoman/lib/repoman/modules/linechecks/base.py +++ /dev/null @@ -1,115 +0,0 @@ -import logging -import re - - -class LineCheck: - """Run a check on a line of an ebuild.""" - - """A regular expression to determine whether to ignore the line""" - ignore_line = False - """True if lines containing nothing more than comments with optional - leading whitespace should be ignored""" - ignore_comment = True - - def __init__(self, errors): - self.errors = errors - - def new(self, pkg): - pass - - def check_eapi(self, eapi): - """Returns if check should be run in the given EAPI (default: True)""" - return True - - def check(self, num, line): - """Run the check on line and return error if there is one""" - if self.re.match(line): - return self.errors[self.error] - - def end(self): - pass - - -class InheritEclass(LineCheck): - """ - Base class for checking for missing inherits, as well as excess inherits. - - Args: - eclass: Set to the name of your eclass. - funcs: A tuple of functions that this eclass provides. - comprehensive: Is the list of functions complete? - exempt_eclasses: If these eclasses are inherited, disable the missing - inherit check. - """ - - def __init__( - self, - eclass, - eclass_eapi_functions, - errors, - funcs=None, - comprehensive=False, - exempt_eclasses=None, - ignore_missing=False, - **kwargs - ): - self._eclass = eclass - self._comprehensive = comprehensive - self._exempt_eclasses = exempt_eclasses - self._ignore_missing = ignore_missing - self.errors = errors - inherit_re = eclass - self._eclass_eapi_functions = eclass_eapi_functions - self._inherit_re = re.compile( - r"^(\s*|.*[|&]\s*)\binherit\s(.*\s)?%s(\s|$)" % inherit_re - ) - # Match when the function is preceded only by leading whitespace, a - # shell operator such as (, {, |, ||, or &&, or optional variable - # setting(s). This prevents false positives in things like elog - # messages, as reported in bug #413285. - logging.debug("InheritEclass, eclass: %s, funcs: %s", eclass, funcs) - self._func_re = re.compile( - r"(^|[|&{(])\s*(\w+=.*)?\b(" + r"|".join(funcs) + r")\b" - ) - - def new(self, pkg): - self.repoman_check_name = "inherit.missing" - # We can't use pkg.inherited because that tells us all the eclasses that - # have been inherited and not just the ones we inherit directly. - self._inherit = False - self._func_call = False - if self._exempt_eclasses is not None: - inherited = pkg.inherited - self._disabled = any(x in inherited for x in self._exempt_eclasses) - else: - self._disabled = False - self._eapi = pkg.eapi - - def check(self, num, line): - if not self._inherit: - self._inherit = self._inherit_re.match(line) - if not self._inherit: - if self._disabled or self._ignore_missing: - return - s = self._func_re.search(line) - if s is not None: - func_name = s.group(3) - eapi_func = self._eclass_eapi_functions.get(func_name) - if eapi_func is None or not eapi_func(self._eapi): - self._func_call = True - return '%s.eclass not inherited, but "%s" called' % ( - self._eclass, - func_name, - ) - elif not self._func_call: - self._func_call = self._func_re.search(line) - - def end(self): - if ( - not self._disabled - and self._comprehensive - and self._inherit - and not self._func_call - ): - self.repoman_check_name = "inherit.unused" - yield "no function called from %s.eclass; please drop" % self._eclass diff --git a/repoman/lib/repoman/modules/linechecks/config.py b/repoman/lib/repoman/modules/linechecks/config.py deleted file mode 100644 index e4275f5ccd8..00000000000 --- a/repoman/lib/repoman/modules/linechecks/config.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: Checks -# Copyright 2007-2017 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains functions used in Repoman to ascertain the quality -and correctness of an ebuild.""" - -import collections -import logging -import os -from copy import deepcopy - -from repoman._portage import portage -from repoman.config import load_config -from repoman import _not_installed - -# Avoid a circular import issue in py2.7 -portage.proxy.lazyimport.lazyimport( - globals(), - "portage.util:stack_lists", -) - - -def merge(dict1, dict2): - """Return a new dictionary by merging two dictionaries recursively.""" - - result = deepcopy(dict1) - - for key, value in dict2.items(): - if isinstance(value, collections.Mapping): - result[key] = merge(result.get(key, {}), value) - else: - result[key] = deepcopy(dict2[key]) - - return result - - -class LineChecksConfig: - """Holds our LineChecks configuration data and operation functions""" - - def __init__(self, repo_settings): - """Class init - - @param repo_settings: RepoSettings instance - @param configpaths: ordered list of filepaths to load - """ - self.repo_settings = repo_settings - self.infopaths = None - self.info_config = None - self._config = None - self.usex_supported_eapis = None - self.in_iuse_supported_eapis = None - self.get_libdir_supported_eapis = None - self.eclass_eapi_functions = {} - self.eclass_export_functions = None - self.eclass_info = {} - self.eclass_info_experimental_inherit = {} - self.errors = {} - self.set_infopaths() - self.load_checks_info() - - def set_infopaths(self): - if _not_installed: - cnfdir = os.path.realpath( - os.path.join( - os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - ) - ), - "cnf/linechecks", - ) - ) - else: - cnfdir = os.path.join( - portage.const.EPREFIX or "/", "usr/share/repoman/linechecks" - ) - repomanpaths = [os.path.join(cnfdir, _file_) for _file_ in os.listdir(cnfdir)] - logging.debug("LineChecksConfig; repomanpaths: %s", repomanpaths) - repopaths = [ - os.path.join(path, "linechecks.yaml") - for path in self.repo_settings.masters_list - ] - self.infopaths = repomanpaths + repopaths - logging.debug("LineChecksConfig; configpaths: %s", self.infopaths) - - def load_checks_info(self, infopaths=None): - """load the config files in order - - @param infopaths: ordered list of filepaths to load - """ - if infopaths: - self.infopaths = infopaths - elif not self.infopaths: - logging.error("LineChecksConfig; Error: No linechecks.yaml files defined") - - configs = load_config( - self.infopaths, "yaml", self.repo_settings.repoman_settings.valid_versions - ) - if configs == {}: - logging.error( - "LineChecksConfig: Failed to load a valid 'linechecks.yaml' file at paths: %s", - self.infopaths, - ) - return False - logging.debug("LineChecksConfig: linechecks.yaml configs: %s", configs) - self.info_config = configs - - self.errors = self.info_config["errors"] - self.usex_supported_eapis = self.info_config.get("usex_supported_eapis", []) - self.in_iuse_supported_eapis = self.info_config.get( - "in_iuse_supported_eapis", [] - ) - self.eclass_info_experimental_inherit = self.info_config.get( - "eclass_info_experimental_inherit", [] - ) - self.get_libdir_supported_eapis = self.in_iuse_supported_eapis - self.eclass_eapi_functions = { - "usex": lambda eapi: eapi not in self.usex_supported_eapis, - "in_iuse": lambda eapi: eapi not in self.in_iuse_supported_eapis, - "get_libdir": lambda eapi: eapi not in self.get_libdir_supported_eapis, - } - - # eclasses that export ${ECLASS}_src_(compile|configure|install) - self.eclass_export_functions = self.info_config.get( - "eclass_export_functions", [] - ) - - self.eclass_info_experimental_inherit = self.info_config.get( - "eclass_info_experimental_inherit", {} - ) - # These are "eclasses are the whole ebuild" type thing. - try: - self.eclass_info_experimental_inherit["eutils"][ - "exempt_eclasses" - ] = self.eclass_export_functions - except KeyError: - pass - try: - self.eclass_info_experimental_inherit["multilib"][ - "exempt_eclasses" - ] = self.eclass_export_functions + [ - "autotools", - "libtool", - "multilib-minimal", - ] - except KeyError: - pass diff --git a/repoman/lib/repoman/modules/linechecks/controller.py b/repoman/lib/repoman/modules/linechecks/controller.py deleted file mode 100644 index f4cfe2aca2b..00000000000 --- a/repoman/lib/repoman/modules/linechecks/controller.py +++ /dev/null @@ -1,164 +0,0 @@ -import logging -import operator -import os -import re - -from repoman.modules.linechecks.base import InheritEclass -from repoman.modules.linechecks.config import LineChecksConfig -from repoman._portage import portage - -# Avoid a circular import issue in py2.7 -portage.proxy.lazyimport.lazyimport( - globals(), - "portage.module:Modules", -) - -MODULES_PATH = os.path.dirname(__file__) -# initial development debug info -logging.debug("LineChecks module path: %s", MODULES_PATH) - - -class LineCheckController: - """Initializes and runs the LineCheck checks""" - - def __init__(self, repo_settings, linechecks): - """Class init - - @param repo_settings: RepoSettings instance - """ - self.repo_settings = repo_settings - self.linechecks = linechecks - self.config = LineChecksConfig(repo_settings) - - self.controller = Modules( - path=MODULES_PATH, namepath="repoman.modules.linechecks" - ) - logging.debug( - "LineCheckController; module_names: %s", self.controller.module_names - ) - - self._constant_checks = None - - self._here_doc_re = re.compile(r".*<<[-]?(\w+)\s*(>\s*\S+\s*)?$") - self._ignore_comment_re = re.compile(r"^\s*#") - self._continuation_re = re.compile(r"(\\)*$") - - def checks_init(self, experimental_inherit=False): - """Initialize the main variables - - @param experimental_inherit boolean - """ - if not experimental_inherit: - # Emulate the old eprefixify.defined and inherit.autotools checks. - self._eclass_info = self.config.eclass_info - else: - self._eclass_info = self.config.eclass_info_experimental_inherit - - self._constant_checks = [] - logging.debug("LineCheckController; modules: %s", self.linechecks) - # Add in the pluggable modules - for mod in self.linechecks: - mod_class = self.controller.get_class(mod) - logging.debug( - "LineCheckController; module_name: %s, class: %s", - mod, - mod_class.__name__, - ) - self._constant_checks.append(mod_class(self.config.errors)) - # Add in the InheritEclass checks - logging.debug( - "LineCheckController; eclass_info.items(): %s", - list(self.config.eclass_info), - ) - for k, kwargs in self.config.eclass_info.items(): - logging.debug("LineCheckController; k: %s, kwargs: %s", k, kwargs) - self._constant_checks.append( - InheritEclass( - k, self.config.eclass_eapi_functions, self.config.errors, **kwargs - ) - ) - - def run_checks(self, contents, pkg): - """Run the configured linechecks - - @param contents: the ebuild contents to check - @param pkg: the package being checked - """ - if self._constant_checks is None: - self.checks_init() - checks = self._constant_checks - here_doc_delim = None - multiline = None - - for lc in checks: - lc.new(pkg) - - multinum = 0 - for num, line in enumerate(contents): - - # Check if we're inside a here-document. - if here_doc_delim is not None: - if here_doc_delim.match(line): - here_doc_delim = None - if here_doc_delim is None: - here_doc = self._here_doc_re.match(line) - if here_doc is not None: - here_doc_delim = re.compile(r"^\s*%s$" % here_doc.group(1)) - if here_doc_delim is not None: - continue - - # Unroll multiline escaped strings so that we can check things: - # inherit foo bar \ - # moo \ - # cow - # This will merge these lines like so: - # inherit foo bar moo cow - # A line ending with an even number of backslashes does not count, - # because the last backslash is escaped. Therefore, search for an - # odd number of backslashes. - line_escaped = ( - operator.sub(*self._continuation_re.search(line).span()) % 2 == 1 - ) - if multiline: - # Chop off the \ and \n bytes from the previous line. - multiline = multiline[:-2] + line - if not line_escaped: - line = multiline - num = multinum - multiline = None - else: - continue - else: - if line_escaped: - multinum = num - multiline = line - continue - - if not line.endswith("#nowarn\n"): - # Finally we have a full line to parse. - is_comment = self._ignore_comment_re.match(line) is not None - for lc in checks: - if is_comment and lc.ignore_comment: - continue - if lc.check_eapi(pkg.eapi): - ignore = lc.ignore_line - if not ignore or not ignore.match(line): - errors = lc.check(num, line) - if errors: - if isinstance(errors, (tuple, list)): - for error in errors: - yield lc.repoman_check_name, "line %d: %s" % ( - num + 1, - error, - ) - else: - yield lc.repoman_check_name, "line %d: %s" % ( - num + 1, - errors, - ) - - for lc in checks: - i = lc.end() - if i is not None: - for e in i: - yield lc.repoman_check_name, e diff --git a/repoman/lib/repoman/modules/linechecks/depend/__init__.py b/repoman/lib/repoman/modules/linechecks/depend/__init__.py deleted file mode 100644 index 7fe35f9686e..00000000000 --- a/repoman/lib/repoman/modules/linechecks/depend/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Depend plug-in module for repoman LineChecks. -Performs dependency checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "depend", - "description": doc, - "provides": { - "implicit-check": { - "name": "implicitdepend", - "sourcefile": "implicit", - "class": "ImplicitRuntimeDeps", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/depend/implicit.py b/repoman/lib/repoman/modules/linechecks/depend/implicit.py deleted file mode 100644 index 5f4c0661869..00000000000 --- a/repoman/lib/repoman/modules/linechecks/depend/implicit.py +++ /dev/null @@ -1,38 +0,0 @@ -import re - -from portage.eapi import eapi_has_implicit_rdepend -from repoman.modules.linechecks.base import LineCheck - - -class ImplicitRuntimeDeps(LineCheck): - """ - Detect the case where DEPEND is set and RDEPEND is unset in the ebuild, - since this triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4). - """ - - repoman_check_name = "RDEPEND.implicit" - _assignment_re = re.compile(r"^\s*(R?DEPEND)\+?=") - - def new(self, pkg): - self._rdepend = False - self._depend = False - - def check_eapi(self, eapi): - # Beginning with EAPI 4, there is no - # implicit RDEPEND=$DEPEND assignment - # to be concerned with. - return eapi_has_implicit_rdepend(eapi) - - def check(self, num, line): - if not self._rdepend: - m = self._assignment_re.match(line) - if m is None: - pass - elif m.group(1) == "RDEPEND": - self._rdepend = True - elif m.group(1) == "DEPEND": - self._depend = True - - def end(self): - if self._depend and not self._rdepend: - yield "RDEPEND is not explicitly assigned" diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/__init__.py b/repoman/lib/repoman/modules/linechecks/deprecated/__init__.py deleted file mode 100644 index 593bc1d3f2a..00000000000 --- a/repoman/lib/repoman/modules/linechecks/deprecated/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Deprecated plug-in module for repoman LineChecks. -Performs miscelaneous deprecation checks on ebuilds not covered by -specialty modules.""" -__doc__ = doc[:] - - -module_spec = { - "name": "deprecated", - "description": doc, - "provides": { - "useq-check": { - "name": "useq", - "sourcefile": "deprecated", - "class": "DeprecatedUseq", - "description": doc, - }, - "hasq-check": { - "name": "hasq", - "sourcefile": "deprecated", - "class": "DeprecatedHasq", - "description": doc, - }, - "preserve-check": { - "name": "preservelib", - "sourcefile": "deprecated", - "class": "PreserveOldLib", - "description": doc, - }, - "bindnow-check": { - "name": "bindnow", - "sourcefile": "deprecated", - "class": "DeprecatedBindnowFlags", - "description": doc, - }, - "inherit-check": { - "name": "inherit", - "sourcefile": "inherit", - "class": "InheritDeprecated", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py b/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py deleted file mode 100644 index abf7dbf423d..00000000000 --- a/repoman/lib/repoman/modules/linechecks/deprecated/deprecated.py +++ /dev/null @@ -1,35 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class DeprecatedUseq(LineCheck): - """Checks for use of the deprecated useq function""" - - repoman_check_name = "ebuild.minorsyn" - re = re.compile(r"(^|.*\b)useq\b") - error = "USEQ_ERROR" - - -class DeprecatedHasq(LineCheck): - """Checks for use of the deprecated hasq function""" - - repoman_check_name = "ebuild.minorsyn" - re = re.compile(r"(^|.*\b)hasq\b") - error = "HASQ_ERROR" - - -class PreserveOldLib(LineCheck): - """Check for calls to the preserve_old_lib function reserved for system packages.""" - - repoman_check_name = "ebuild.minorsyn" - re = re.compile(r".*preserve_old_lib") - error = "PRESERVE_OLD_LIB" - - -class DeprecatedBindnowFlags(LineCheck): - """Check for calls to the deprecated bindnow-flags function.""" - - repoman_check_name = "ebuild.minorsyn" - re = re.compile(r".*\$\(bindnow-flags\)") - error = "DEPRECATED_BINDNOW_FLAGS" diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py deleted file mode 100644 index 7d7f66d5d6c..00000000000 --- a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py +++ /dev/null @@ -1,67 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class InheritDeprecated(LineCheck): - """Check if ebuild directly or indirectly inherits a deprecated eclass.""" - - repoman_check_name = "inherit.deprecated" - - # deprecated eclass : new eclass (False if no new eclass) - deprecated_eclasses = { - "autotools-multilib": "multilib-minimal", - "autotools-utils": False, - "base": False, - "bash-completion": "bash-completion-r1", - "boost-utils": False, - "clutter": "gnome2", - "cmake-utils": "cmake", - "confutils": False, - "distutils": "distutils-r1", - "epatch": "(eapply since EAPI 6)", - "fdo-mime": "xdg-utils", - "games": False, - "gems": "ruby-fakegem", - "git-2": "git-r3", - "gpe": False, - "gst-plugins-bad": "gstreamer", - "gst-plugins-base": "gstreamer", - "gst-plugins-good": "gstreamer", - "gst-plugins-ugly": "gstreamer", - "gst-plugins10": "gstreamer", - "ltprune": False, - "mono": "mono-env", - "python": "python-r1 / python-single-r1 / python-any-r1", - "ruby": "ruby-ng", - "user": "GLEP 81", - "versionator": "eapi7-ver (built-in since EAPI 7)", - "x-modular": "xorg-2", - "xfconf": False, - } - - _inherit_re = re.compile(r"^\s*inherit\s(.*)$") - - def check(self, num, line): - direct_inherits = None - m = self._inherit_re.match(line) - if m is not None: - direct_inherits = m.group(1) - if direct_inherits: - direct_inherits = direct_inherits.split() - - if not direct_inherits: - return - - errors = [] - for eclass in direct_inherits: - replacement = self.deprecated_eclasses.get(eclass) - if replacement is None: - pass - elif replacement is False: - errors.append("please migrate from " "'%s' (no replacement)" % eclass) - else: - errors.append( - "please migrate from " "'%s' to '%s'" % (eclass, replacement) - ) - return errors diff --git a/repoman/lib/repoman/modules/linechecks/do/__init__.py b/repoman/lib/repoman/modules/linechecks/do/__init__.py deleted file mode 100644 index 597b961e798..00000000000 --- a/repoman/lib/repoman/modules/linechecks/do/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Do plug-in module for repoman LineChecks. -Performs do* checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "nonrelative-check": { - "name": "dosym", - "sourcefile": "dosym", - "class": "EbuildNonRelativeDosym", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/do/dosym.py b/repoman/lib/repoman/modules/linechecks/do/dosym.py deleted file mode 100644 index 2276b3315e3..00000000000 --- a/repoman/lib/repoman/modules/linechecks/do/dosym.py +++ /dev/null @@ -1,20 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildNonRelativeDosym(LineCheck): - """Check ebuild for dosym using absolute paths instead of relative.""" - - repoman_check_name = "ebuild.absdosym" - variables = ("D", "ED", "ROOT", "EROOT", "BROOT") - regex = re.compile( - r'^\s*dosym\s+(["\']?((\$(%s)\W|\${(%s)(%%/)?})|/(bin|etc|lib|opt|sbin|srv|usr|var))\S*)' - % ("|".join(variables), "|".join(variables)), - getattr(re, "ASCII", 0), - ) - - def check(self, num, line): - match = self.regex.match(line) - if match: - return "dosym '%s'... could use relative path" % match.group(1) diff --git a/repoman/lib/repoman/modules/linechecks/eapi/__init__.py b/repoman/lib/repoman/modules/linechecks/eapi/__init__.py deleted file mode 100644 index 68e6b722a0f..00000000000 --- a/repoman/lib/repoman/modules/linechecks/eapi/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eapi plug-in module for repoman LineChecks. -Performs eapi dependant checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "eapi", - "description": doc, - "provides": { - "definition-check": { - "name": "definition", - "sourcefile": "definition", - "class": "EapiDefinition", - "description": doc, - }, - "srcprepare-check": { - "name": "srcprepare", - "sourcefile": "checks", - "class": "UndefinedSrcPrepareSrcConfigurePhases", - "description": doc, - }, - "eapi3deprecated-check": { - "name": "eapi3deprecated", - "sourcefile": "checks", - "class": "Eapi3DeprecatedFuncs", - "description": doc, - }, - "pkgpretend-check": { - "name": "pkgpretend", - "sourcefile": "checks", - "class": "UndefinedPkgPretendPhase", - "description": doc, - }, - "eapi4incompatible-check": { - "name": "eapi4incompatible", - "sourcefile": "checks", - "class": "Eapi4IncompatibleFuncs", - "description": doc, - }, - "eapi4gonevars-check": { - "name": "eapi4gonevars", - "sourcefile": "checks", - "class": "Eapi4GoneVars", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/eapi/checks.py b/repoman/lib/repoman/modules/linechecks/eapi/checks.py deleted file mode 100644 index 64bb509a7d4..00000000000 --- a/repoman/lib/repoman/modules/linechecks/eapi/checks.py +++ /dev/null @@ -1,79 +0,0 @@ -import re - -from portage.eapi import ( - eapi_has_src_prepare_and_src_configure, - eapi_has_dosed_dohard, - eapi_exports_AA, - eapi_has_pkg_pretend, -) -from repoman.modules.linechecks.base import LineCheck - - -# EAPI <2 checks -class UndefinedSrcPrepareSrcConfigurePhases(LineCheck): - repoman_check_name = "EAPI.incompatible" - src_configprepare_re = re.compile(r"\s*(src_configure|src_prepare)\s*\(\)") - - def check_eapi(self, eapi): - return not eapi_has_src_prepare_and_src_configure(eapi) - - def check(self, num, line): - m = self.src_configprepare_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + " phase is not defined in EAPI < 2" - - -# EAPI-3 checks -class Eapi3DeprecatedFuncs(LineCheck): - repoman_check_name = "EAPI.deprecated" - deprecated_commands_re = re.compile(r"^\s*(check_license)\b") - - def check_eapi(self, eapi): - return eapi not in ("0", "1", "2") - - def check(self, num, line): - m = self.deprecated_commands_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + " has been deprecated in EAPI=3" - - -# EAPI <4 checks -class UndefinedPkgPretendPhase(LineCheck): - repoman_check_name = "EAPI.incompatible" - pkg_pretend_re = re.compile(r"\s*(pkg_pretend)\s*\(\)") - - def check_eapi(self, eapi): - return not eapi_has_pkg_pretend(eapi) - - def check(self, num, line): - m = self.pkg_pretend_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + " phase is not defined in EAPI < 4" - - -# EAPI-4 checks -class Eapi4IncompatibleFuncs(LineCheck): - repoman_check_name = "EAPI.incompatible" - banned_commands_re = re.compile(r"^\s*(dosed|dohard)") - - def check_eapi(self, eapi): - return not eapi_has_dosed_dohard(eapi) - - def check(self, num, line): - m = self.banned_commands_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + " has been banned in EAPI=4" - - -class Eapi4GoneVars(LineCheck): - repoman_check_name = "EAPI.incompatible" - undefined_vars_re = re.compile(r".*\$(\{(AA|KV|EMERGE_FROM)\}|(AA|KV|EMERGE_FROM))") - - def check_eapi(self, eapi): - # AA, KV, and EMERGE_FROM should not be referenced in EAPI 4 or later. - return not eapi_exports_AA(eapi) - - def check(self, num, line): - m = self.undefined_vars_re.match(line) - if m is not None: - return ("variable '$%s'" % m.group(1)) + " is gone in EAPI=4" diff --git a/repoman/lib/repoman/modules/linechecks/eapi/definition.py b/repoman/lib/repoman/modules/linechecks/eapi/definition.py deleted file mode 100644 index 964fa4ea340..00000000000 --- a/repoman/lib/repoman/modules/linechecks/eapi/definition.py +++ /dev/null @@ -1,35 +0,0 @@ -from repoman.modules.linechecks.base import LineCheck -from repoman._portage import portage - - -class EapiDefinition(LineCheck): - """ - Check that EAPI assignment conforms to PMS section 7.3.1 - (first non-comment, non-blank line). - """ - - repoman_check_name = "EAPI.definition" - ignore_comment = True - _eapi_re = portage._pms_eapi_re - - def new(self, pkg): - self._cached_eapi = pkg.eapi - self._parsed_eapi = None - self._eapi_line_num = None - - def check(self, num, line): - if self._eapi_line_num is None and line.strip(): - self._eapi_line_num = num + 1 - m = self._eapi_re.match(line) - if m is not None: - self._parsed_eapi = m.group(2) - - def end(self): - if self._parsed_eapi is None: - if self._cached_eapi != "0": - yield "valid EAPI assignment must occur on or before line: %s" % self._eapi_line_num - elif self._parsed_eapi != self._cached_eapi: - yield ( - "bash returned EAPI '%s' which does not match " - "assignment on line: %s" % (self._cached_eapi, self._eapi_line_num) - ) diff --git a/repoman/lib/repoman/modules/linechecks/emake/__init__.py b/repoman/lib/repoman/modules/linechecks/emake/__init__.py deleted file mode 100644 index 9a2c54915be..00000000000 --- a/repoman/lib/repoman/modules/linechecks/emake/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Emake plug-in module for repoman LineChecks. -Performs emake checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "paralleldisabled-check": { - "name": "paralleldisabled", - "sourcefile": "emake", - "class": "EMakeParallelDisabledViaMAKEOPTS", - "description": doc, - }, - "autodefault-check": { - "name": "autodefault", - "sourcefile": "emake", - "class": "WantAutoDefaultValue", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/emake/emake.py b/repoman/lib/repoman/modules/linechecks/emake/emake.py deleted file mode 100644 index 3bb7fc35b98..00000000000 --- a/repoman/lib/repoman/modules/linechecks/emake/emake.py +++ /dev/null @@ -1,25 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EMakeParallelDisabledViaMAKEOPTS(LineCheck): - """Check for MAKEOPTS=-j1 that disables parallelization.""" - - repoman_check_name = "upstream.workaround" - re = re.compile(r'^\s*MAKEOPTS=(\'|")?.*-j\s*1\b') - error = "EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS" - - -class WantAutoDefaultValue(LineCheck): - """Check setting WANT_AUTO* to latest (default value).""" - - repoman_check_name = "ebuild.minorsyn" - _re = re.compile(r'^WANT_AUTO(CONF|MAKE)=(\'|")?latest') - - def check(self, num, line): - m = self._re.match(line) - if m is not None: - return ( - "WANT_AUTO" + m.group(1) + ' redundantly set to default value "latest"' - ) diff --git a/repoman/lib/repoman/modules/linechecks/gentoo_header/__init__.py b/repoman/lib/repoman/modules/linechecks/gentoo_header/__init__.py deleted file mode 100644 index 565b43ec2f5..00000000000 --- a/repoman/lib/repoman/modules/linechecks/gentoo_header/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Gentoo-header plug-in module for repoman LineChecks. -Performs header checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "header-check": { - "name": "gentooheader", - "sourcefile": "header", - "class": "EbuildHeader", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/gentoo_header/header.py b/repoman/lib/repoman/modules/linechecks/gentoo_header/header.py deleted file mode 100644 index 7ef77ae4483..00000000000 --- a/repoman/lib/repoman/modules/linechecks/gentoo_header/header.py +++ /dev/null @@ -1,56 +0,0 @@ -import re -import time - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildHeader(LineCheck): - """Ensure ebuilds have proper headers - Copyright header errors - CVS header errors - License header errors - - Args: - modification_year - Year the ebuild was last modified - """ - - repoman_check_name = "ebuild.badheader" - - gentoo_copyright = r"^# Copyright ((1999|2\d\d\d)-)?(?P<year2>%s) (?P<author>.*)$" - gentoo_license = ( - "# Distributed under the terms" " of the GNU General Public License v2" - ) - id_header_re = re.compile(r".*\$(Id|Header)(:.*)?\$.*") - blank_line_re = re.compile(r"^$") - ignore_comment = False - - def new(self, pkg): - if pkg.mtime is None: - self.modification_year = r"2\d\d\d" - else: - self.modification_year = str(time.gmtime(pkg.mtime)[0]) - self.gentoo_copyright_re = re.compile( - self.gentoo_copyright % self.modification_year - ) - - def check(self, num, line): - if num > 2: - return - elif num == 0: - match = self.gentoo_copyright_re.match(line) - if match is None: - return self.errors["COPYRIGHT_ERROR"] - if not ( - match.group("author") == "Gentoo Authors" - or ( - int(match.group("year2")) < 2019 - and match.group("author") == "Gentoo Foundation" - ) - ): - return self.errors["COPYRIGHT_ERROR"] - elif num == 1 and line.rstrip("\n") != self.gentoo_license: - return self.errors["LICENSE_ERROR"] - elif num == 2 and self.id_header_re.match(line): - return self.errors["ID_HEADER_ERROR"] - elif num == 2 and not self.blank_line_re.match(line): - return self.errors["NO_BLANK_LINE_ERROR"] diff --git a/repoman/lib/repoman/modules/linechecks/helpers/__init__.py b/repoman/lib/repoman/modules/linechecks/helpers/__init__.py deleted file mode 100644 index b65b454f1ec..00000000000 --- a/repoman/lib/repoman/modules/linechecks/helpers/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Helpers plug-in module for repoman LineChecks. -Performs variable helpers checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "nooffset-check": { - "name": "nooffset", - "sourcefile": "offset", - "class": "NoOffsetWithHelpers", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/helpers/offset.py b/repoman/lib/repoman/modules/linechecks/helpers/offset.py deleted file mode 100644 index 72ece48d585..00000000000 --- a/repoman/lib/repoman/modules/linechecks/helpers/offset.py +++ /dev/null @@ -1,21 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class NoOffsetWithHelpers(LineCheck): - """Check that the image location, the alternate root offset, and the - offset prefix (D, ROOT, ED, EROOT and EPREFIX) are not used with - helpers""" - - repoman_check_name = "variable.usedwithhelpers" - # Ignore matches in quoted strings like this: - # elog "installed into ${ROOT}usr/share/php5/apc/." - _install_funcs = ( - "docinto|do(compress|dir|hard)" "|exeinto|fowners|fperms|insinto|into" - ) - _quoted_vars = "D|ROOT|ED|EROOT|EPREFIX" - re = re.compile( - r'^[^#"\']*\b(%s)\s+"?\$\{?(%s)\b.*' % (_install_funcs, _quoted_vars) - ) - error = "NO_OFFSET_WITH_HELPERS" diff --git a/repoman/lib/repoman/modules/linechecks/nested/__init__.py b/repoman/lib/repoman/modules/linechecks/nested/__init__.py deleted file mode 100644 index 83d69127dde..00000000000 --- a/repoman/lib/repoman/modules/linechecks/nested/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Nested plug-in module for repoman LineChecks. -Performs nested subshell checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "nesteddie-check": { - "name": "nesteddie", - "sourcefile": "nested", - "class": "EbuildNestedDie", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/nested/nested.py b/repoman/lib/repoman/modules/linechecks/nested/nested.py deleted file mode 100644 index fc04bde0165..00000000000 --- a/repoman/lib/repoman/modules/linechecks/nested/nested.py +++ /dev/null @@ -1,14 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildNestedDie(LineCheck): - """Check ebuild for nested die statements (die statements in subshells)""" - - repoman_check_name = "ebuild.nesteddie" - nesteddie_re = re.compile(r"^[^#]*\s\(\s[^)]*\bdie\b") - - def check(self, num, line): - if self.nesteddie_re.match(line): - return self.errors["NESTED_DIE_ERROR"] diff --git a/repoman/lib/repoman/modules/linechecks/nested/nesteddie.py b/repoman/lib/repoman/modules/linechecks/nested/nesteddie.py deleted file mode 100644 index 4b256bb1fd4..00000000000 --- a/repoman/lib/repoman/modules/linechecks/nested/nesteddie.py +++ /dev/null @@ -1,9 +0,0 @@ -class EbuildNestedDie(LineCheck): - """Check ebuild for nested die statements (die statements in subshells)""" - - repoman_check_name = "ebuild.nesteddie" - nesteddie_re = re.compile(r"^[^#]*\s\(\s[^)]*\bdie\b") - - def check(self, num, line): - if self.nesteddie_re.match(line): - return errors.NESTED_DIE_ERROR diff --git a/repoman/lib/repoman/modules/linechecks/patches/__init__.py b/repoman/lib/repoman/modules/linechecks/patches/__init__.py deleted file mode 100644 index e4aec16a135..00000000000 --- a/repoman/lib/repoman/modules/linechecks/patches/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Patches plug-in module for repoman LineChecks. -Performs PATCHES variable checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "patches-check": { - "name": "patches", - "sourcefile": "patches", - "class": "EbuildPatches", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/patches/patches.py b/repoman/lib/repoman/modules/linechecks/patches/patches.py deleted file mode 100644 index ca9a4cada6d..00000000000 --- a/repoman/lib/repoman/modules/linechecks/patches/patches.py +++ /dev/null @@ -1,22 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildPatches(LineCheck): - """Ensure ebuilds use bash arrays for PATCHES to ensure white space safety""" - - repoman_check_name = "ebuild.patches" - re = re.compile(r"^\s*PATCHES=[^\(]") - error = "PATCHES_ERROR" - - def check_eapi(self, eapi): - return eapi in ( - "0", - "1", - "2", - "3", - "4", - "4-slot-abi", - "5", - ) diff --git a/repoman/lib/repoman/modules/linechecks/phases/__init__.py b/repoman/lib/repoman/modules/linechecks/phases/__init__.py deleted file mode 100644 index 880eac02b28..00000000000 --- a/repoman/lib/repoman/modules/linechecks/phases/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Phases plug-in module for repoman LineChecks. -Performs phase dependant checks on ebuilds using a PhaseCheck base class. -""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "emakeparallel-check": { - "name": "emakeparallel", - "sourcefile": "phase", - "class": "EMakeParallelDisabled", - "description": doc, - }, - "srccompileeconf-check": { - "name": "srccompileeconf", - "sourcefile": "phase", - "class": "SrcCompileEconf", - "description": doc, - }, - "srcunpackpatches-check": { - "name": "srcunpackpatches", - "sourcefile": "phase", - "class": "SrcUnpackPatches", - "description": doc, - }, - "pmsvariablerefphasescope-check": { - "name": "pmsvariablerefphasescope", - "sourcefile": "phase", - "class": "PMSVariableReference", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/phases/phase.py b/repoman/lib/repoman/modules/linechecks/phases/phase.py deleted file mode 100644 index 5d89b263d9f..00000000000 --- a/repoman/lib/repoman/modules/linechecks/phases/phase.py +++ /dev/null @@ -1,188 +0,0 @@ -import fnmatch -import re -import types - -from portage.eapi import ( - eapi_has_broot, - eapi_has_sysroot, - eapi_has_src_prepare_and_src_configure, - eapi_exports_AA, - eapi_exports_replace_vars, - eapi_exports_ECLASSDIR, - eapi_exports_PORTDIR, - eapi_supports_prefix, - eapi_exports_merge_type, -) -from repoman.modules.linechecks.base import LineCheck - - -class PhaseCheck(LineCheck): - """basic class for function detection""" - - func_end_re = re.compile(r"^\}$") - phase_funcs = ( - "pkg_pretend", - "pkg_setup", - "src_unpack", - "src_prepare", - "src_configure", - "src_compile", - "src_test", - "src_install", - "pkg_preinst", - "pkg_postinst", - "pkg_prerm", - "pkg_postrm", - "pkg_config", - ) - phases_re = re.compile("(%s)" % "|".join(phase_funcs)) - in_phase = "" - - def check(self, num, line): - m = self.phases_re.match(line) - if m is not None: - self.in_phase = m.group(1) - if self.in_phase != "" and self.func_end_re.match(line) is not None: - self.in_phase = "" - - return self.phase_check(num, line) - - def phase_check(self, num, line): - """override this function for your checks""" - pass - - -class EMakeParallelDisabled(PhaseCheck): - """Check for emake -j1 calls which disable parallelization.""" - - repoman_check_name = "upstream.workaround" - re = re.compile(r"^\s*emake\s+.*-j\s*1\b") - - def phase_check(self, num, line): - if self.in_phase == "src_compile" or self.in_phase == "src_install": - if self.re.match(line): - return self.errors["EMAKE_PARALLEL_DISABLED"] - - -class SrcCompileEconf(PhaseCheck): - repoman_check_name = "ebuild.minorsyn" - configure_re = re.compile(r"\s(econf|./configure)") - - def check_eapi(self, eapi): - return eapi_has_src_prepare_and_src_configure(eapi) - - def phase_check(self, num, line): - if self.in_phase == "src_compile": - m = self.configure_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + " call should be moved to src_configure" - - -class SrcUnpackPatches(PhaseCheck): - repoman_check_name = "ebuild.minorsyn" - src_prepare_tools_re = re.compile(r"\s(e?patch|sed)\s") - - def check_eapi(self, eapi): - return eapi_has_src_prepare_and_src_configure(eapi) - - def phase_check(self, num, line): - if self.in_phase == "src_unpack": - m = self.src_prepare_tools_re.search(line) - if m is not None: - return ("'%s'" % m.group(1)) + " call should be moved to src_prepare" - - -# Refererences -# - https://projects.gentoo.org/pms/7/pms.html#x1-10900011.1 -# - https://pkgcore.github.io/pkgcheck/_modules/pkgcheck/checks/codingstyle.html#VariableScopeCheck -_pms_vars = ( - ("A", None, ("src_*", "pkg_nofetch")), - ("AA", eapi_exports_AA, ("src_*", "pkg_nofetch")), - ("FILESDIR", None, ("src_*",)), - ("DISTDIR", None, ("src_*",)), - ("WORKDIR", None, ("src_*",)), - ("S", None, ("src_*",)), - ("PORTDIR", eapi_exports_PORTDIR, ("src_*",)), - ("ECLASSDIR", eapi_exports_ECLASSDIR, ("src_*",)), - ("ROOT", None, ("pkg_*",)), - ("EROOT", eapi_supports_prefix, ("pkg_*",)), - ("SYSROOT", eapi_has_sysroot, ("src_*", "pkg_setup")), - ("ESYSROOT", eapi_has_sysroot, ("src_*", "pkg_setup")), - ("BROOT", eapi_has_broot, ("src_*", "pkg_setup")), - ("D", None, ("src_install", "pkg_preinst", "pkg_postint")), - ("ED", eapi_supports_prefix, ("src_install", "pkg_preinst", "pkg_postint")), - ("DESTTREE", None, ("src_install",)), - ("INSDESTTREE", None, ("src_install",)), - ("MERGE_TYPE", eapi_exports_merge_type, ("pkg_*",)), - ("REPLACING_VERSIONS", eapi_exports_replace_vars, ("pkg_*",)), - ("REPLACED_BY_VERSION", eapi_exports_replace_vars, ("pkg_prerm", "pkg_postrm")), -) - - -def _compile_phases(): - phase_vars = {} - for phase_func in PhaseCheck.phase_funcs: - for variable, eapi_filter, allowed_scopes in _pms_vars: - allowed = False - for scope in allowed_scopes: - if fnmatch.fnmatch(phase_func, scope): - allowed = True - break - - if not allowed: - phase_vars.setdefault(phase_func, []).append((variable, eapi_filter)) - - phase_info = {} - for phase_func, prohibited_vars in phase_vars.items(): - phase_func_vars = [] - for variable, eapi_filter in prohibited_vars: - phase_func_vars.append(variable) - phase_obj = phase_info[phase_func] = types.SimpleNamespace() - phase_obj.prohibited_vars = dict(prohibited_vars) - phase_obj.var_names = "(%s)" % "|".join( - variable for variable, eapi_filter in prohibited_vars - ) - phase_obj.var_reference = re.compile( - r"\$(\{|)%s(\}|\W)" % (phase_obj.var_names,) - ) - - return phase_info - - -class PMSVariableReference(PhaseCheck): - """Check phase scope for references to variables specified by PMS""" - - repoman_check_name = "variable.phase" - phase_info = _compile_phases() - - def new(self, pkg): - self._eapi = pkg.eapi - - def end(self): - self._eapi = None - - def phase_check(self, num, line): - try: - phase_info = self.phase_info[self.in_phase] - except KeyError: - return - - eapi = self._eapi - issues = [] - for m in phase_info.var_reference.finditer(line): - open_brace = m.group(1) - var_name = m.group(2) - close_brace = m.group(3) - # discard \W if matched by (\}|\W) - close_brace = close_brace if close_brace == "}" else "" - if bool(open_brace) != bool(close_brace): - continue - var_name = m.group(2) - eapi_filter = phase_info.prohibited_vars[var_name] - if eapi_filter is not None and not eapi_filter(eapi): - continue - issues.append( - "phase %s: EAPI %s: variable %s: Forbidden reference to variable specified by PMS" - % (self.in_phase, eapi, var_name) - ) - return issues diff --git a/repoman/lib/repoman/modules/linechecks/portage/__init__.py b/repoman/lib/repoman/modules/linechecks/portage/__init__.py deleted file mode 100644 index d30877d6dc0..00000000000 --- a/repoman/lib/repoman/modules/linechecks/portage/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Portage plug-in module for repoman LineChecks. -Performs checks for internal portage variable usage in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "internal-check": { - "name": "portageinternal", - "sourcefile": "internal", - "class": "PortageInternal", - "description": doc, - }, - "portageinternalvariableassignment-check": { - "name": "portageinternalvariableassignment", - "sourcefile": "internal", - "class": "PortageInternalVariableAssignment", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/portage/internal.py b/repoman/lib/repoman/modules/linechecks/portage/internal.py deleted file mode 100644 index 5d6af108847..00000000000 --- a/repoman/lib/repoman/modules/linechecks/portage/internal.py +++ /dev/null @@ -1,32 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class PortageInternal(LineCheck): - repoman_check_name = "portage.internal" - ignore_comment = True - # Match when the command is preceded only by leading whitespace or a shell - # operator such as (, {, |, ||, or &&. This prevents false positives in - # things like elog messages, as reported in bug #413285. - - internal_portage_func_or_var = ( - "ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib" - ) - re = re.compile(r"^(\s*|.*[|&{(]+\s*)\b(%s)\b" % internal_portage_func_or_var) - - def check(self, num, line): - """Run the check on line and return error if there is one""" - m = self.re.match(line) - if m is not None: - return "'%s' called" % m.group(2) - - -class PortageInternalVariableAssignment(LineCheck): - repoman_check_name = "portage.internal" - internal_assignment = re.compile(r"\s*(export\s+)?(EXTRA_ECONF|EXTRA_EMAKE)\+?=") - - def check(self, num, line): - match = self.internal_assignment.match(line) - if match is not None: - return "Assignment to variable %s" % match.group(2) diff --git a/repoman/lib/repoman/modules/linechecks/quotes/__init__.py b/repoman/lib/repoman/modules/linechecks/quotes/__init__.py deleted file mode 100644 index deb4ced974b..00000000000 --- a/repoman/lib/repoman/modules/linechecks/quotes/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Nested plug-in module for repoman LineChecks. -Performs nested subshell checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "quote-check": { - "name": "quote", - "sourcefile": "quotes", - "class": "EbuildQuote", - "description": doc, - }, - "quoteda-check": { - "name": "quoteda", - "sourcefile": "quoteda", - "class": "EbuildQuotedA", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/quotes/quoteda.py b/repoman/lib/repoman/modules/linechecks/quotes/quoteda.py deleted file mode 100644 index f44d4e38eb3..00000000000 --- a/repoman/lib/repoman/modules/linechecks/quotes/quoteda.py +++ /dev/null @@ -1,15 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildQuotedA(LineCheck): - """Ensure ebuilds have no quoting around ${A}""" - - repoman_check_name = "ebuild.minorsyn" - a_quoted = re.compile(r".*\"\$(\{A\}|A)\"") - - def check(self, num, line): - match = self.a_quoted.match(line) - if match: - return 'Quoted "${A}"' diff --git a/repoman/lib/repoman/modules/linechecks/quotes/quotes.py b/repoman/lib/repoman/modules/linechecks/quotes/quotes.py deleted file mode 100644 index c170c10ef39..00000000000 --- a/repoman/lib/repoman/modules/linechecks/quotes/quotes.py +++ /dev/null @@ -1,92 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildQuote(LineCheck): - """Ensure ebuilds have valid quoting around things like D,FILESDIR, etc...""" - - repoman_check_name = "ebuild.minorsyn" - _message_commands = ["die", "echo", "eerror", "einfo", "elog", "eqawarn", "ewarn"] - _message_re = re.compile(r"\s(" + "|".join(_message_commands) + r')\s+"[^"]*"\s*$') - _ignored_commands = ["local", "export"] + _message_commands - ignore_line = re.compile( - r"(^$)|(^\s*#.*)|(^\s*\w+=.*)" - + r"|(^\s*(" - + "|".join(_ignored_commands) - + r")\s+)" - ) - ignore_comment = False - var_names = ["D", "DISTDIR", "FILESDIR", "S", "T", "ROOT", "BROOT", "WORKDIR"] - - # EAPI=3/Prefix vars - var_names += ["ED", "EPREFIX", "EROOT"] - - # variables for games.eclass - var_names += [ - "Ddir", - "GAMES_PREFIX_OPT", - "GAMES_DATADIR", - "GAMES_DATADIR_BASE", - "GAMES_SYSCONFDIR", - "GAMES_STATEDIR", - "GAMES_LOGDIR", - "GAMES_BINDIR", - ] - - # variables for multibuild.eclass - var_names += ["BUILD_DIR"] - - var_names = "(%s)" % "|".join(var_names) - var_reference = re.compile(r"\$(\{%s\}|%s\W)" % (var_names, var_names)) - missing_quotes = re.compile(r'(\s|^)[^"\'\s]*\$\{?%s\}?[^"\'\s]*(\s|$)' % var_names) - cond_begin = re.compile(r"(^|\s+)\[\[($|\\$|\s+)") - cond_end = re.compile(r"(^|\s+)\]\]($|\\$|\s+)") - - def check(self, num, line): - if self.var_reference.search(line) is None: - return - # There can be multiple matches / violations on a single line. We - # have to make sure none of the matches are violators. Once we've - # found one violator, any remaining matches on the same line can - # be ignored. - pos = 0 - while pos <= len(line) - 1: - missing_quotes = self.missing_quotes.search(line, pos) - if not missing_quotes: - break - # If the last character of the previous match is a whitespace - # character, that character may be needed for the next - # missing_quotes match, so search overlaps by 1 character. - group = missing_quotes.group() - pos = missing_quotes.end() - 1 - - # Filter out some false positives that can - # get through the missing_quotes regex. - if self.var_reference.search(group) is None: - continue - - # Filter matches that appear to be an - # argument to a message command. - # For example: false || ewarn "foo $WORKDIR/bar baz" - message_match = self._message_re.search(line) - if ( - message_match is not None - and message_match.start() < pos - and message_match.end() > pos - ): - break - - # This is an attempt to avoid false positives without getting - # too complex, while possibly allowing some (hopefully - # unlikely) violations to slip through. We just assume - # everything is correct if the there is a ' [[ ' or a ' ]] ' - # anywhere in the whole line (possibly continued over one - # line). - if self.cond_begin.search(line) is not None: - continue - if self.cond_end.search(line) is not None: - continue - - # Any remaining matches on the same line can be ignored. - return self.errors["MISSING_QUOTES_ERROR"] diff --git a/repoman/lib/repoman/modules/linechecks/uri/__init__.py b/repoman/lib/repoman/modules/linechecks/uri/__init__.py deleted file mode 100644 index 4018020c3f3..00000000000 --- a/repoman/lib/repoman/modules/linechecks/uri/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Uri plug-in module for repoman LineChecks. -Performs HOMEPAGE variable checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "httpsuri-check": { - "name": "httpsuri", - "sourcefile": "uri", - "class": "UriUseHttps", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/uri/uri.py b/repoman/lib/repoman/modules/linechecks/uri/uri.py deleted file mode 100644 index 9f3e49fe0c4..00000000000 --- a/repoman/lib/repoman/modules/linechecks/uri/uri.py +++ /dev/null @@ -1,30 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class UriUseHttps(LineCheck): - """Check that we use https:// for known good sites.""" - - repoman_check_name = "uri.https" - _SITES = ( - r"([-._a-zA-Z0-9]*\.)?apache\.org", - r"((alioth|packages(\.qa)?|people|www)\.)?debian\.org", - # Most FDO sites support https, but not all (like tango). - # List the most common ones here for now. - r"((anongit|bugs|cgit|dri|patchwork|people|specifications|www|xcb|xorg)\.)?freedesktop\.org", - r"((bugs|dev|wiki|www)\.)?gentoo\.org", - r"((wiki)\.)?github\.(io|com)", - r"savannah\.(non)?gnu\.org", - r"((gcc|www)\.)?gnu\.org", - r"curl\.haxx\.se", - r"((bugzilla|git|mirrors|patchwork|planet|www(\.wiki)?)\.)?kernel\.org", - r"((bugs|wiki|www)\.)?linuxfoundation\.org", - r"((docs|pypi|www)\.)?python\.org", - r"(sf|sourceforge)\.net", - r"(www\.)?(enlightenment|sourceware|x)\.org", - ) - # Try to anchor the end of the URL so we don't get false positives - # with http://github.com.foo.bar.com/. Unlikely, but possible. - re = re.compile(r'.*\bhttp://(%s)(\s|["\'/]|$)' % r"|".join(_SITES)) - error = "URI_HTTPS" diff --git a/repoman/lib/repoman/modules/linechecks/use/__init__.py b/repoman/lib/repoman/modules/linechecks/use/__init__.py deleted file mode 100644 index 0d738c37c72..00000000000 --- a/repoman/lib/repoman/modules/linechecks/use/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Use plug-in module for repoman LineChecks. -Performs Built-With-Use checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "builtwith-check": { - "name": "builtwith", - "sourcefile": "builtwith", - "class": "BuiltWithUse", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/use/builtwith.py b/repoman/lib/repoman/modules/linechecks/use/builtwith.py deleted file mode 100644 index 0b16de02802..00000000000 --- a/repoman/lib/repoman/modules/linechecks/use/builtwith.py +++ /dev/null @@ -1,9 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class BuiltWithUse(LineCheck): - repoman_check_name = "ebuild.minorsyn" - re = re.compile(r"(^|.*\b)built_with_use\b") - error = "BUILT_WITH_USE" diff --git a/repoman/lib/repoman/modules/linechecks/useless/__init__.py b/repoman/lib/repoman/modules/linechecks/useless/__init__.py deleted file mode 100644 index 9be9515841d..00000000000 --- a/repoman/lib/repoman/modules/linechecks/useless/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Useless plug-in module for repoman LineChecks. -Performs checks for useless operations on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "uselesscds-check": { - "name": "uselesscds", - "sourcefile": "cd", - "class": "EbuildUselessCdS", - "description": doc, - }, - "uselessdodoc-check": { - "name": "uselessdodoc", - "sourcefile": "dodoc", - "class": "EbuildUselessDodoc", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/useless/cd.py b/repoman/lib/repoman/modules/linechecks/useless/cd.py deleted file mode 100644 index 73063209ffd..00000000000 --- a/repoman/lib/repoman/modules/linechecks/useless/cd.py +++ /dev/null @@ -1,24 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildUselessCdS(LineCheck): - """Check for redundant cd ${S} statements""" - - repoman_check_name = "ebuild.minorsyn" - _src_phases = r"^\s*src_(prepare|configure|compile|install|test)\s*\(\)" - method_re = re.compile(_src_phases) - cds_re = re.compile(r'^\s*cd\s+("\$(\{S\}|S)"|\$(\{S\}|S))\s') - - def __init__(self, errors): - self.errors = errors - self.check_next_line = False - - def check(self, num, line): - if self.check_next_line: - self.check_next_line = False - if self.cds_re.match(line): - return self.errors["REDUNDANT_CD_S_ERROR"] - elif self.method_re.match(line): - self.check_next_line = True diff --git a/repoman/lib/repoman/modules/linechecks/useless/dodoc.py b/repoman/lib/repoman/modules/linechecks/useless/dodoc.py deleted file mode 100644 index 49f6fde9185..00000000000 --- a/repoman/lib/repoman/modules/linechecks/useless/dodoc.py +++ /dev/null @@ -1,17 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildUselessDodoc(LineCheck): - """Check ebuild for useless files in dodoc arguments.""" - - repoman_check_name = "ebuild.minorsyn" - uselessdodoc_re = re.compile( - r"^\s*dodoc(\s+|\s+.*\s+)(ABOUT-NLS|COPYING|LICENCE|LICENSE)($|\s)" - ) - - def check(self, num, line): - match = self.uselessdodoc_re.match(line) - if match: - return "Useless dodoc '%s'" % match.group(2) diff --git a/repoman/lib/repoman/modules/linechecks/whitespace/__init__.py b/repoman/lib/repoman/modules/linechecks/whitespace/__init__.py deleted file mode 100644 index 17b3abab500..00000000000 --- a/repoman/lib/repoman/modules/linechecks/whitespace/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Whitespace plug-in module for repoman LineChecks. -Performs checks for useless whitespace in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "whitespace-check": { - "name": "whitespace", - "sourcefile": "whitespace", - "class": "EbuildWhitespace", - "description": doc, - }, - "blankline-check": { - "name": "blankline", - "sourcefile": "blank", - "class": "EbuildBlankLine", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/whitespace/blank.py b/repoman/lib/repoman/modules/linechecks/whitespace/blank.py deleted file mode 100644 index 89e38c68cb4..00000000000 --- a/repoman/lib/repoman/modules/linechecks/whitespace/blank.py +++ /dev/null @@ -1,24 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildBlankLine(LineCheck): - repoman_check_name = "ebuild.minorsyn" - ignore_comment = False - blank_line = re.compile(r"^$") - - def new(self, pkg): - self.line_is_blank = False - - def check(self, num, line): - if self.line_is_blank and self.blank_line.match(line): - return "Useless blank line" - if self.blank_line.match(line): - self.line_is_blank = True - else: - self.line_is_blank = False - - def end(self): - if self.line_is_blank: - yield "Useless blank line on last line" diff --git a/repoman/lib/repoman/modules/linechecks/whitespace/whitespace.py b/repoman/lib/repoman/modules/linechecks/whitespace/whitespace.py deleted file mode 100644 index a1274048d4c..00000000000 --- a/repoman/lib/repoman/modules/linechecks/whitespace/whitespace.py +++ /dev/null @@ -1,20 +0,0 @@ -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildWhitespace(LineCheck): - """Ensure ebuilds have proper whitespacing""" - - repoman_check_name = "ebuild.minorsyn" - - ignore_line = re.compile(r"(^$)|(^(\t)*#)") - ignore_comment = False - leading_spaces = re.compile(r"^[\S\t]") - trailing_whitespace = re.compile(r".*([\S]$)") - - def check(self, num, line): - if self.leading_spaces.match(line) is None: - return self.errors["LEADING_SPACES_ERROR"] - if self.trailing_whitespace.match(line) is None: - return self.errors["TRAILING_WHITESPACE_ERROR"] diff --git a/repoman/lib/repoman/modules/linechecks/workaround/__init__.py b/repoman/lib/repoman/modules/linechecks/workaround/__init__.py deleted file mode 100644 index eff33cf2339..00000000000 --- a/repoman/lib/repoman/modules/linechecks/workaround/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Workaround plug-in module for repoman LineChecks. -Performs checks for upstream workarounds in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "do", - "description": doc, - "provides": { - "noasneeded-check": { - "name": "noasneeded", - "sourcefile": "workarounds", - "class": "NoAsNeeded", - "description": doc, - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/linechecks/workaround/workarounds.py b/repoman/lib/repoman/modules/linechecks/workaround/workarounds.py deleted file mode 100644 index 47b6881ffaa..00000000000 --- a/repoman/lib/repoman/modules/linechecks/workaround/workarounds.py +++ /dev/null @@ -1,11 +0,0 @@ -import re # pylint: disable=unused-import - -from repoman.modules.linechecks.base import LineCheck - - -class NoAsNeeded(LineCheck): - """Check for calls to the no-as-needed function.""" - - repoman_check_name = "upstream.workaround" - re = re.compile(r".*\$\(no-as-needed\)") - error = "NO_AS_NEEDED" diff --git a/repoman/lib/repoman/modules/scan/__init__.py b/repoman/lib/repoman/modules/scan/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/modules/scan/__init__.py +++ /dev/null diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py deleted file mode 100644 index cac382b6f2b..00000000000 --- a/repoman/lib/repoman/modules/scan/depend/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Depend plug-in module for repoman. -Performs Dependency checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "depend", - "description": doc, - "provides": { - "profile-module": { - "name": "profile", - "sourcefile": "profile", - "class": "ProfileDependsChecks", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": [ - "qatracker", - "portdb", - "profiles", - "options", - "repo_metadata", - "repo_settings", - "include_arches", - "include_profiles", - "caches", - "repoman_incrementals", - "env", - "have", - "dev_keywords", - ], - "func_kwargs": { - "ebuild": (None, None), - "pkg": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/depend/_depend_checks.py b/repoman/lib/repoman/modules/scan/depend/_depend_checks.py deleted file mode 100644 index 11a2fbf56d6..00000000000 --- a/repoman/lib/repoman/modules/scan/depend/_depend_checks.py +++ /dev/null @@ -1,260 +0,0 @@ -# -*- coding:utf-8 -*- - -import collections - -from _emerge.Package import Package - -from portage.dep import Atom - -from repoman.check_missingslot import check_missingslot - -# import our initialized portage instance -from repoman._portage import portage - - -def check_slotop(depstr, is_valid_flag, badsyntax, mytype, qatracker, relative_path): - """Checks if RDEPEND uses ':=' slot operator - in '||' style dependencies.""" - - try: - # to find use of ':=' in '||' we preserve - # tree structure of dependencies - my_dep_tree = portage.dep.use_reduce( - depstr, - flat=False, - matchall=1, - is_valid_flag=is_valid_flag, - opconvert=True, - token_class=portage.dep.Atom, - ) - except portage.exception.InvalidDependString as e: - my_dep_tree = None - badsyntax.append((mytype, str(e))) - - def _traverse_tree(dep_tree, in_any_of): - # leaf - if isinstance(dep_tree, Atom): - atom = dep_tree - if in_any_of and atom.slot_operator == "=": - qatracker.add_error( - "dependency.badslotop", - "%s: %s: '%s' uses ':=' slot operator under '||' dep clause." - % (relative_path, mytype, atom), - ) - - # branches - if isinstance(dep_tree, list): - if len(dep_tree) == 0: - return - # entering any-of - if dep_tree[0] == "||": - _traverse_tree(dep_tree[1:], in_any_of=True) - else: - for branch in dep_tree: - _traverse_tree(branch, in_any_of=in_any_of) - - _traverse_tree(my_dep_tree, False) - - -def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata, qadata): - """Checks the ebuild dependencies for errors - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_metadata: dictionary of various repository items. - @returns: (unknown_pkgs, badlicsyntax) - """ - - unknown_pkgs = set() - - inherited_java_eclass = ( - "java-pkg-2" in ebuild.inherited or "java-pkg-opt-2" in ebuild.inherited - ) - inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited - # operator_tokens = set(["||", "(", ")"]) - badsyntax = [] - for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES"): - mydepstr = ebuild.metadata[mytype] - - buildtime = mytype in Package._buildtime_keys - runtime = mytype in Package._runtime_keys - token_class = None - if mytype.endswith("DEPEND"): - token_class = portage.dep.Atom - - try: - atoms = portage.dep.use_reduce( - mydepstr, - matchall=1, - flat=True, - is_valid_flag=pkg.iuse.is_valid_flag, - token_class=token_class, - ) - except portage.exception.InvalidDependString as e: - atoms = None - badsyntax.append((mytype, str(e))) - - if atoms and mytype.endswith("DEPEND"): - if runtime and "test?" in mydepstr.split(): - qatracker.add_error( - mytype + ".suspect", - "%s: 'test?' USE conditional in %s" - % (ebuild.relative_path, mytype), - ) - - for atom in atoms: - if atom == "||": - continue - - is_blocker = atom.blocker - - # Skip dependency.unknown for blockers, so that we - # don't encourage people to remove necessary blockers, - # as discussed in bug 382407. We use atom.without_use - # due to bug 525376. - if ( - not is_blocker - and not portdb.xmatch("match-all", atom.without_use) - and not atom.cp.startswith("virtual/") - ): - unknown_pkgs.add((mytype, atom.unevaluated_atom)) - - if not atom.blocker: - all_deprecated = False - for pkg_match in portdb.xmatch("match-all", atom): - if any( - repo_metadata["package.deprecated"].iterAtomsForPackage( - pkg_match - ) - ): - all_deprecated = True - else: - all_deprecated = False - break - - if all_deprecated: - qatracker.add_error( - "dependency.deprecated", - ebuild.relative_path + ": '%s'" % atom, - ) - - if pkg.category != "virtual": - if not is_blocker and atom.cp in qadata.suspect_virtual: - qatracker.add_error( - "virtual.suspect", - ebuild.relative_path - + ": %s: consider using '%s' instead of '%s'" - % (mytype, qadata.suspect_virtual[atom.cp], atom), - ) - if not is_blocker and atom.cp.startswith("perl-core/"): - qatracker.add_error( - "dependency.perlcore", - ebuild.relative_path - + ": %s: please use '%s' instead of '%s'" - % ( - mytype, - atom.replace("perl-core/", "virtual/perl-"), - atom, - ), - ) - - if ( - buildtime - and not is_blocker - and not inherited_java_eclass - and atom.cp == "virtual/jdk" - ): - qatracker.add_error("java.eclassesnotused", ebuild.relative_path) - elif ( - buildtime - and not is_blocker - and not inherited_wxwidgets_eclass - and atom.cp == "x11-libs/wxGTK" - ): - qatracker.add_error( - "wxwidgets.eclassnotused", - "%s: %ss on x11-libs/wxGTK without inheriting" - " wxwidgets.eclass" % (ebuild.relative_path, mytype), - ) - elif runtime: - if not is_blocker and atom.cp in qadata.suspect_rdepend: - qatracker.add_error( - mytype + ".suspect", ebuild.relative_path + ": '%s'" % atom - ) - - if ( - atom.operator == "~" - and portage.versions.catpkgsplit(atom.cpv)[3] != "r0" - ): - qacat = "dependency.badtilde" - qatracker.add_error( - qacat, - "%s: %s uses the ~ operator" - " with a non-zero revision: '%s'" - % (ebuild.relative_path, mytype, atom), - ) - # plain =foo-1.2.3 without revision or * - if atom.operator == "=" and "-r" not in atom.version: - qacat = "dependency.equalsversion" - qatracker.add_error( - qacat, - "%s: %s uses the = operator with" - " no revision: '%s'; if any revision is" - " acceptable, use '~' instead; if only -r0" - " then please append '-r0' to the dep" - % (ebuild.relative_path, mytype, atom), - ) - - check_missingslot( - atom, - mytype, - ebuild.eapi, - portdb, - qatracker, - ebuild.relative_path, - ebuild.metadata, - ) - - if runtime: - check_slotop( - mydepstr, - pkg.iuse.is_valid_flag, - badsyntax, - mytype, - qatracker, - ebuild.relative_path, - ) - - baddepsyntax = False - dedup = collections.defaultdict(set) - for m, b in badsyntax: - if b in dedup[m]: - continue - dedup[m].add(b) - - if m.endswith("DEPEND"): - baddepsyntax = True - qacat = "dependency.syntax" - else: - qacat = m + ".syntax" - qatracker.add_error(qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) - - # Parse the LICENSE variable, remove USE conditions and flatten it. - licenses = portage.dep.use_reduce(ebuild.metadata["LICENSE"], matchall=1, flat=True) - - # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. - for lic in licenses: - # Need to check for "||" manually as no portage - # function will remove it without removing values. - if lic not in repo_metadata["liclist"] and lic != "||": - qatracker.add_error( - "LICENSE.invalid", "%s: %s" % (ebuild.relative_path, lic) - ) - elif lic in repo_metadata["lic_deprecated"]: - qatracker.add_error( - "LICENSE.deprecated", "%s: %s" % (ebuild.relative_path, lic) - ) - - return unknown_pkgs, baddepsyntax diff --git a/repoman/lib/repoman/modules/scan/depend/_gen_arches.py b/repoman/lib/repoman/modules/scan/depend/_gen_arches.py deleted file mode 100644 index 6e3000014b2..00000000000 --- a/repoman/lib/repoman/modules/scan/depend/_gen_arches.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding:utf-8 -*- - - -def _gen_arches(ebuild, options, repo_settings, profiles): - """Determines the arches for the ebuild following the profile rules - - @param ebuild: Ebuild which we check (object). - @param profiles: dictionary - @param options: cli options - @param repo_settings: repository settings instance - @returns: dictionary, including arches set - """ - if options.ignore_arches: - arches = [ - [ - repo_settings.repoman_settings["ARCH"], - repo_settings.repoman_settings["ARCH"], - repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split(), - ] - ] - else: - arches = set() - for keyword in ebuild.keywords: - if keyword[0] == "-": - continue - elif keyword[0] == "~": - arch = keyword[1:] - if arch == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add( - ( - keyword, - expanded_arch, - (expanded_arch, "~" + expanded_arch), - ) - ) - else: - arches.add((keyword, arch, (arch, keyword))) - else: - # For ebuilds with stable keywords, check if the - # dependencies are satisfiable for unstable - # configurations, since use.stable.mask is not - # applied for unstable configurations (see bug - # 563546). - if keyword == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add((keyword, expanded_arch, (expanded_arch,))) - arches.add( - ( - keyword, - expanded_arch, - (expanded_arch, "~" + expanded_arch), - ) - ) - else: - arches.add((keyword, keyword, (keyword,))) - arches.add((keyword, keyword, (keyword, "~" + keyword))) - if not arches: - # Use an empty profile for checking dependencies of - # packages that have empty KEYWORDS. - arches.add(("**", "**", ("**",))) - - return arches diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py deleted file mode 100644 index a0cb3a31437..00000000000 --- a/repoman/lib/repoman/modules/scan/depend/profile.py +++ /dev/null @@ -1,427 +0,0 @@ -# -*- coding:utf-8 -*- - - -import copy -import functools -import os -import types -from pprint import pformat - -from _emerge.Package import Package - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase -from repoman.modules.scan.depend._depend_checks import _depend_checks -from repoman.modules.scan.depend._gen_arches import _gen_arches -from portage.dep import Atom -from portage.package.ebuild.profile_iuse import iter_iuse_vars -from portage.util import getconfig -from portage.util.futures import asyncio -from portage.util.futures.executor.fork import ForkExecutor -from portage.util.futures.iter_completed import async_iter_completed - - -def sort_key(item): - return item[2].sub_path - - -class ProfileDependsChecks(ScanBase): - """Perform dependency checks for the different profiles""" - - def __init__(self, **kwargs): - """Class init - - @param qatracker: QATracker instance - @param portdb: portdb instance - @param profiles: dictionary - @param options: cli options - @param repo_settings: repository settings instance - @param include_arches: set - @param include_profiles: set - @param caches: dictionary of our caches - @param repoman_incrementals: tuple - @param env: the environment - @param have: dictionary instance - @param dev_keywords: developer profile keywords - @param repo_metadata: dictionary of various repository items. - """ - self.qatracker = kwargs.get("qatracker") - self.portdb = kwargs.get("portdb") - self.profiles = kwargs.get("profiles") - self.options = kwargs.get("options") - self.repo_settings = kwargs.get("repo_settings") - self.include_arches = kwargs.get("include_arches") - self.include_profiles = kwargs.get("include_profiles") - self.caches = kwargs.get("caches") - self.repoman_incrementals = kwargs.get("repoman_incrementals") - self.env = kwargs.get("env") - self.have = kwargs.get("have") - self.dev_keywords = kwargs.get("dev_keywords") - self.repo_metadata = kwargs.get("repo_metadata") - - def check(self, **kwargs): - """Perform profile dependant dependency checks - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - """ - ebuild = kwargs.get("ebuild").get() - pkg = kwargs.get("pkg").get() - - ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks( - ebuild, - pkg, - self.portdb, - self.qatracker, - self.repo_metadata, - self.repo_settings.qadata, - ) - - relevant_profiles = [] - for keyword, arch, groups in _gen_arches( - ebuild, self.options, self.repo_settings, self.profiles - ): - if arch not in self.profiles: - # A missing profile will create an error further down - # during the KEYWORDS verification. - continue - - if self.include_arches is not None: - if arch not in self.include_arches: - continue - - for prof in self.profiles[arch]: - if self.include_profiles is not None: - if prof.sub_path not in self.include_profiles: - continue - relevant_profiles.append((keyword, groups, prof)) - - relevant_profiles.sort(key=sort_key) - ebuild.relevant_profiles = relevant_profiles - - if self.options.jobs <= 1: - for task in self._iter_tasks(None, None, ebuild, pkg): - task, results = task - for result in results: - self._check_result(task, result) - - loop = asyncio.get_event_loop() - loop.run_until_complete(self._async_check(loop, **kwargs)) - - return False - - async def _async_check(self, loop, **kwargs): - """Perform async profile dependant dependency checks - - @param arches: - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param baddepsyntax: boolean - @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) - @returns: dictionary - """ - ebuild = kwargs.get("ebuild").get() - pkg = kwargs.get("pkg").get() - unknown_pkgs = ebuild.unknown_pkgs - baddepsyntax = ebuild.baddepsyntax - - # Use max_workers=True to ensure immediate fork, since _iter_tasks - # needs the fork to create a snapshot of current state. - executor = ForkExecutor(max_workers=self.options.jobs) - - if self.options.jobs > 1: - for future_done_set in async_iter_completed( - self._iter_tasks(loop, executor, ebuild, pkg), - max_jobs=self.options.jobs, - max_load=self.options.load_average, - ): - for task in await future_done_set: - task, results = task.result() - for result in results: - self._check_result(task, result) - - if not baddepsyntax and unknown_pkgs: - type_map = {} - for mytype, atom in unknown_pkgs: - type_map.setdefault(mytype, set()).add(atom) - for mytype, atoms in type_map.items(): - self.qatracker.add_error( - "dependency.unknown", - "%s: %s: %s" - % (ebuild.relative_path, mytype, ", ".join(sorted(atoms))), - ) - - async def _task(self, task): - await task.future - return (task, task.future.result()) - - def _iter_tasks(self, loop, executor, ebuild, pkg): - for keyword, groups, prof in ebuild.relevant_profiles: - - is_stable_profile = prof.status == "stable" - is_dev_profile = prof.status == "dev" and self.options.include_dev - is_exp_profile = ( - prof.status == "exp" and self.options.include_exp_profiles == "y" - ) - if not (is_stable_profile or is_dev_profile or is_exp_profile): - continue - - dep_settings = self.caches["arch"].get(prof.sub_path) - if dep_settings is None: - dep_settings = portage.config( - config_profile_path=prof.abs_path, - config_incrementals=self.repoman_incrementals, - config_root=self.repo_settings.config_root, - local_config=False, - _unmatched_removal=self.options.unmatched_removal, - env=self.env, - repositories=self.repo_settings.repoman_settings.repositories, - ) - - if not prof.abs_path: - self._populate_implicit_iuse( - dep_settings, self.repo_settings.repo_config.eclass_db.porttrees - ) - - dep_settings.categories = self.repo_settings.repoman_settings.categories - if self.options.without_mask: - dep_settings._mask_manager_obj = copy.deepcopy( - dep_settings._mask_manager - ) - dep_settings._mask_manager._pmaskdict.clear() - self.caches["arch"][prof.sub_path] = dep_settings - - xmatch_cache_key = (prof.sub_path, tuple(groups)) - xcache = self.caches["arch_xmatch"].get(xmatch_cache_key) - if xcache is None: - self.portdb.melt() - self.portdb.freeze() - xcache = self.portdb.xcache - xcache.update(self.caches["shared_xmatch"]) - self.caches["arch_xmatch"][xmatch_cache_key] = xcache - - self.repo_settings.trees[self.repo_settings.root][ - "porttree" - ].settings = dep_settings - self.portdb.settings = dep_settings - self.portdb.xcache = xcache - - dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) - # just in case, prevent config.reset() from nuking these. - dep_settings.backup_changes("ACCEPT_KEYWORDS") - - # This attribute is used in dbapi._match_use() to apply - # use.stable.{mask,force} settings based on the stable - # status of the parent package. This is required in order - # for USE deps of unstable packages to be resolved correctly, - # since otherwise use.stable.{mask,force} settings of - # dependencies may conflict (see bug #456342). - dep_settings._parent_stable = dep_settings._isStable(pkg) - - # Handle package.use*.{force,mask) calculation, for use - # in dep_check. - dep_settings.useforce = dep_settings._use_manager.getUseForce( - pkg, stable=dep_settings._parent_stable - ) - dep_settings.usemask = dep_settings._use_manager.getUseMask( - pkg, stable=dep_settings._parent_stable - ) - - task = types.SimpleNamespace(ebuild=ebuild, prof=prof, keyword=keyword) - - target = functools.partial(self._task_subprocess, task, pkg, dep_settings) - - if self.options.jobs <= 1: - yield (task, target()) - else: - task.future = asyncio.ensure_future( - loop.run_in_executor(executor, target), loop=loop - ) - yield asyncio.ensure_future(self._task(task), loop=loop) - - def _task_subprocess(self, task, pkg, dep_settings): - ebuild = task.ebuild - baddepsyntax = ebuild.baddepsyntax - results = [] - prof = task.prof - if not baddepsyntax: - ismasked = not ebuild.archs or pkg.cpv not in self.portdb.xmatch( - "match-visible", - Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name)), - ) - if ismasked: - if not self.have["pmasked"]: - self.have["pmasked"] = bool( - dep_settings._getMaskAtom(pkg.cpv, ebuild.metadata) - ) - if self.options.ignore_masked: - return results - # we are testing deps for a masked package; give it some lee-way - suffix = "masked" - matchmode = "minimum-all-ignore-profile" - else: - suffix = "" - matchmode = "minimum-visible" - - if not self.have["dev_keywords"]: - self.have["dev_keywords"] = bool( - self.dev_keywords.intersection(ebuild.keywords) - ) - - if prof.status == "dev": - suffix = suffix + "indev" - elif prof.status == "exp": - suffix = suffix + "inexp" - - for mytype in Package._dep_keys: - - mykey = "dependency.bad" + suffix - myvalue = ebuild.metadata[mytype] - if not myvalue: - continue - - success, atoms = portage.dep_check( - myvalue, - self.portdb, - dep_settings, - use="all", - mode=matchmode, - trees=self.repo_settings.trees, - ) - - results.append( - types.SimpleNamespace( - atoms=atoms, success=success, mykey=mykey, mytype=mytype - ) - ) - - return results - - def _check_result(self, task, result): - prof = task.prof - keyword = task.keyword - ebuild = task.ebuild - unknown_pkgs = ebuild.unknown_pkgs - - success = result.success - atoms = result.atoms - mykey = result.mykey - mytype = result.mytype - - if success: - if atoms: - - # Don't bother with dependency.unknown for - # cases in which *DEPEND.bad is triggered. - for atom in atoms: - # dep_check returns all blockers and they - # aren't counted for *DEPEND.bad, so we - # ignore them here. - if not atom.blocker: - unknown_pkgs.discard((mytype, atom.unevaluated_atom)) - - if not prof.sub_path: - # old-style virtuals currently aren't - # resolvable with empty profile, since - # 'virtuals' mappings are unavailable - # (it would be expensive to search - # for PROVIDE in all ebuilds) - atoms = [ - atom - for atom in atoms - if not ( - atom.cp.startswith("virtual/") - and not self.portdb.cp_list(atom.cp) - ) - ] - - # we have some unsolvable deps - # remove ! deps, which always show up as unsatisfiable - all_atoms = [ - str(atom.unevaluated_atom) for atom in atoms if not atom.blocker - ] - - # if we emptied out our list, continue: - if not all_atoms: - return - - # Filter out duplicates. We do this by hand (rather - # than use a set) so the order is stable and better - # matches the order that's in the ebuild itself. - atoms = [] - for atom in all_atoms: - if atom not in atoms: - atoms.append(atom) - - if self.options.output_style in ["column"]: - self.qatracker.add_error( - mykey, - "%s: %s: %s(%s) %s" - % (ebuild.relative_path, mytype, keyword, prof, repr(atoms)), - ) - else: - self.qatracker.add_error( - mykey, - "%s: %s: %s(%s)\n%s" - % ( - ebuild.relative_path, - mytype, - keyword, - prof, - pformat(atoms, indent=6), - ), - ) - else: - if self.options.output_style in ["column"]: - self.qatracker.add_error( - mykey, - "%s: %s: %s(%s) %s" - % (ebuild.relative_path, mytype, keyword, prof, repr(atoms)), - ) - else: - self.qatracker.add_error( - mykey, - "%s: %s: %s(%s)\n%s" - % ( - ebuild.relative_path, - mytype, - keyword, - prof, - pformat(atoms, indent=6), - ), - ) - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) - - @staticmethod - def _populate_implicit_iuse(config, repo_locations): - """ - Populate implicit IUSE for the empty profile, see bug 660982. - - @param config: config instance for the empty profile - @type config: portage.config - @param repo_locations: locations of repositories containing relevant - implicit IUSE settings - @type repo_locations: list - """ - dest = config.configdict["defaults"] - for location in repo_locations: - for parent_dir, dirs, files in os.walk(os.path.join(location, "profiles")): - src = getconfig(os.path.join(parent_dir, "make.defaults")) - if not src: - continue - for k, v in iter_iuse_vars(src): - v_before = dest.get(k) - if v_before is not None: - merged_values = set(v_before.split()) - merged_values.update(v.split()) - v = " ".join(sorted(merged_values)) - dest[k] = v - - config.regenerate() - config._init_iuse() diff --git a/repoman/lib/repoman/modules/scan/directories/__init__.py b/repoman/lib/repoman/modules/scan/directories/__init__.py deleted file mode 100644 index 68596267d16..00000000000 --- a/repoman/lib/repoman/modules/scan/directories/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Directories plug-in module for repoman. -Performs an FilesChecks check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "directories", - "description": doc, - "provides": { - "directories-module": { - "name": "files", - "sourcefile": "files", - "class": "FileChecks", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": [ - "portdb", - "qatracker", - "repo_settings", - "vcs_settings", - ], - "func_kwargs": { - "changed": (None, None), - "checkdir": (None, None), - "checkdirlist": (None, None), - "checkdir_relative": (None, None), - }, - "module_runsIn": ["pkgs"], - }, - "mtime-module": { - "name": "mtime", - "sourcefile": "mtime", - "class": "MtimeChecks", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": [ - "vcs_settings", - ], - "func_kwargs": { - "changed": (None, None), - "ebuild": (None, None), - "pkg": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/directories/files.py b/repoman/lib/repoman/modules/scan/directories/files.py deleted file mode 100644 index 246d8848333..00000000000 --- a/repoman/lib/repoman/modules/scan/directories/files.py +++ /dev/null @@ -1,99 +0,0 @@ -# -*- coding:utf-8 -*- - -"""repoman/checks/diretories/files.py - -""" - -import io - -from portage import _encodings, _unicode_encode -from portage import os - -from repoman.modules.vcs.vcs import vcs_new_changed -from repoman.modules.scan.scanbase import ScanBase - - -class FileChecks(ScanBase): - """Performs various file checks in the package's directory""" - - def __init__(self, **kwargs): - """ - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: settings instance - @param vcs_settings: VCSSettings instance - """ - super(FileChecks, self).__init__(**kwargs) - self.portdb = kwargs.get("portdb") - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - self.repoman_settings = self.repo_settings.repoman_settings - self.vcs_settings = kwargs.get("vcs_settings") - - def check(self, **kwargs): - """Checks the ebuild sources and files for errors - - @param checkdir: string, directory path - @param checkdir_relative: repolevel determined path - @param changed: dictionary instance - @returns: dictionary - """ - checkdir = kwargs.get("checkdir") - checkdirlist = kwargs.get("checkdirlist").get() - checkdir_relative = kwargs.get("checkdir_relative") - changed = kwargs.get("changed").changed - new = kwargs.get("changed").new - for y_file in checkdirlist: - index = self.repo_settings.repo_config.find_invalid_path_char(y_file) - if index != -1: - y_relative = os.path.join(checkdir_relative, y_file) - invcs = self.vcs_settings.vcs is not None - inchangeset = vcs_new_changed(y_relative, changed, new) - if invcs and not inchangeset: - # If the file isn't in the VCS new or changed set, then - # assume that it's an irrelevant temporary file (Manifest - # entries are not generated for file names containing - # prohibited characters). See bug #406877. - index = -1 - if index != -1: - self.qatracker.add_error( - "file.name", "%s/%s: char '%s'" % (checkdir, y_file, y_file[index]) - ) - - if not ( - y_file in ("ChangeLog", "metadata.xml") or y_file.endswith(".ebuild") - ): - continue - f = None - try: - line = 1 - f = io.open( - _unicode_encode( - os.path.join(checkdir, y_file), - encoding=_encodings["fs"], - errors="strict", - ), - mode="r", - encoding=_encodings["repo.content"], - ) - for l in f: - line += 1 - except UnicodeDecodeError as ue: - s = ue.object[: ue.start] - l2 = s.count("\n") - line += l2 - if l2 != 0: - s = s[s.rfind("\n") + 1 :] - self.qatracker.add_error( - "file.UTF8", - "%s/%s: line %i, just after: '%s'" % (checkdir, y_file, line, s), - ) - finally: - if f is not None: - f.close() - return False - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/directories/mtime.py b/repoman/lib/repoman/modules/scan/directories/mtime.py deleted file mode 100644 index 34e21025705..00000000000 --- a/repoman/lib/repoman/modules/scan/directories/mtime.py +++ /dev/null @@ -1,30 +0,0 @@ -from repoman.modules.scan.scanbase import ScanBase - - -class MtimeChecks(ScanBase): - def __init__(self, **kwargs): - self.vcs_settings = kwargs.get("vcs_settings") - - def check(self, **kwargs): - """Perform a changelog and untracked checks on the ebuild - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param changed: dictionary instance - @returns: dictionary - """ - ebuild = kwargs.get("ebuild").get() - changed = kwargs.get("changed") - pkg = kwargs.get("pkg").get() - if not self.vcs_settings.vcs_preserves_mtime: - if ( - ebuild.ebuild_path not in changed.new_ebuilds - and ebuild.ebuild_path not in changed.ebuilds - ): - pkg.mtime = None - return False - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/eapi/__init__.py b/repoman/lib/repoman/modules/scan/eapi/__init__.py deleted file mode 100644 index a86d74b3c92..00000000000 --- a/repoman/lib/repoman/modules/scan/eapi/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eapi plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "eapi", - "description": doc, - "provides": { - "live-module": { - "name": "eapi", - "sourcefile": "eapi", - "class": "EAPIChecks", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": ["qatracker", "repo_settings"], - "func_kwargs": { - "ebuild": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/eapi/eapi.py b/repoman/lib/repoman/modules/scan/eapi/eapi.py deleted file mode 100644 index df7c220e644..00000000000 --- a/repoman/lib/repoman/modules/scan/eapi/eapi.py +++ /dev/null @@ -1,50 +0,0 @@ -"""eapi.py -Perform checks on the EAPI variable. -""" - -from repoman.modules.scan.scanbase import ScanBase - - -class EAPIChecks(ScanBase): - """Perform checks on the EAPI variable.""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - @param repo_settings: Repository settings - """ - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - - def check(self, **kwargs): - """ - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - """ - ebuild = kwargs.get("ebuild").get() - - if not self._checkBanned(ebuild): - self._checkDeprecated(ebuild) - return False - - def _checkBanned(self, ebuild): - if self.repo_settings.repo_config.eapi_is_banned(ebuild.eapi): - self.qatracker.add_error( - "repo.eapi-banned", "%s: %s" % (ebuild.relative_path, ebuild.eapi) - ) - return True - return False - - def _checkDeprecated(self, ebuild): - if self.repo_settings.repo_config.eapi_is_deprecated(ebuild.eapi): - self.qatracker.add_error( - "repo.eapi-deprecated", "%s: %s" % (ebuild.relative_path, ebuild.eapi) - ) - return True - return False - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/ebuild/__init__.py b/repoman/lib/repoman/modules/scan/ebuild/__init__.py deleted file mode 100644 index 029b5628dd1..00000000000 --- a/repoman/lib/repoman/modules/scan/ebuild/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Ebuild plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "ebuild", - "description": doc, - "provides": { - "ebuild-module": { - "name": "ebuild", - "sourcefile": "ebuild", - "class": "Ebuild", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": [ - "qatracker", - "repo_settings", - "vcs_settings", - "checks", - "portdb", - ], - "func_kwargs": { - "can_force": (None, None), - "catdir": (None, None), - "changed": (None, None), - "changelog_modified": (None, None), - "checkdir": (None, None), - "checkdirlist": (None, None), - "ebuild": ("Future", "UNSET"), - "pkg": ("Future", "UNSET"), - "pkgdir": (None, None), - "pkgs": ("Future", "dict"), - "repolevel": (None, None), - "validity_future": (None, None), - "xpkg": (None, None), - "y_ebuild": (None, None), - }, - "module_runsIn": ["pkgs", "ebuilds"], - }, - "multicheck-module": { - "name": "multicheck", - "sourcefile": "multicheck", - "class": "MultiCheck", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": [ - "qatracker", - "options", - "repo_settings", - "linechecks", - ], - "func_kwargs": { - "ebuild": (None, None), - "pkg": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/ebuild/ebuild.py b/repoman/lib/repoman/modules/scan/ebuild/ebuild.py deleted file mode 100644 index ae327578004..00000000000 --- a/repoman/lib/repoman/modules/scan/ebuild/ebuild.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding:utf-8 -*- - -import re -import stat - -from _emerge.Package import Package -from _emerge.RootConfig import RootConfig - -from repoman.modules.scan.scanbase import ScanBase - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.exception import InvalidPackageName - -pv_toolong_re = re.compile(r"[0-9]{19,}") - - -class Ebuild(ScanBase): - """Class to run primary checks on ebuilds""" - - def __init__(self, **kwargs): - """Class init - - @param qatracker: QATracker instance - @param portdb: portdb instance - @param repo_settings: repository settings instance - @param vcs_settings: VCSSettings instance - @param checks: checks dictionary - """ - super(Ebuild, self).__init__(**kwargs) - self.qatracker = kwargs.get("qatracker") - self.portdb = kwargs.get("portdb") - self.repo_settings = kwargs.get("repo_settings") - self.vcs_settings = kwargs.get("vcs_settings") - self.checks = kwargs.get("checks") - self.root_config = RootConfig( - self.repo_settings.repoman_settings, - self.repo_settings.trees[self.repo_settings.root], - None, - ) - self.changed = None - self.xpkg = None - self.y_ebuild = None - self.pkg = None - self.metadata = None - self.eapi = None - self.inherited = None - self.live_ebuild = None - self.keywords = None - self.pkgs = {} - - def _set_paths(self, **kwargs): - repolevel = kwargs.get("repolevel") - self.relative_path = os.path.join(self.xpkg, self.y_ebuild + ".ebuild") - self.full_path = os.path.join(self.repo_settings.repodir, self.relative_path) - self.ebuild_path = self.y_ebuild + ".ebuild" - if repolevel < 3: - self.ebuild_path = os.path.join(kwargs.get("pkgdir"), self.ebuild_path) - if repolevel < 2: - self.ebuild_path = os.path.join(kwargs.get("catdir"), self.ebuild_path) - self.ebuild_path = os.path.join(".", self.ebuild_path) - - @property - def untracked(self): - """Determines and returns if the ebuild is not tracked by the vcs""" - do_check = self.vcs_settings.vcs in ("cvs", "svn", "bzr") - really_notadded = ( - self.checks["ebuild_notadded"] - and self.y_ebuild not in self.vcs_settings.eadded - ) - if do_check and really_notadded: - # ebuild not added to vcs - return True - return False - - def check(self, **kwargs): - """Perform a changelog and untracked checks on the ebuild - - @param xpkg: Package in which we check (object). - @param y_ebuild: Ebuild which we check (string). - @param changed: dictionary instance - @param repolevel: The depth within the repository - @param catdir: The category directiory - @param pkgdir: the package directory - @returns: dictionary, including {ebuild object} - """ - self.xpkg = kwargs.get("xpkg") - self.y_ebuild = kwargs.get("y_ebuild") - self.changed = kwargs.get("changed") - changelog_modified = kwargs.get("changelog_modified") - self._set_paths(**kwargs) - - if ( - self.checks["changelog"] - and not changelog_modified - and self.ebuild_path in self.changed.new_ebuilds - ): - self.qatracker.add_error("changelog.ebuildadded", self.relative_path) - - if self.untracked: - # ebuild not added to vcs - self.qatracker.add_error( - "ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild" - ) - # update the dynamic data - dyn_ebuild = kwargs.get("ebuild") - dyn_ebuild.set(self) - return False - - def set_pkg_data(self, **kwargs): - """Sets some classwide data needed for some of the checks - - @returns: dictionary - """ - self.pkg = self.pkgs[self.y_ebuild] - self.metadata = self.pkg._metadata - self.eapi = self.metadata["EAPI"] - self.inherited = self.pkg.inherited - self.live_ebuild = "live" in self.metadata["PROPERTIES"].split() - self.keywords = self.metadata["KEYWORDS"].split() - self.archs = set( - kw.lstrip("~") for kw in self.keywords if not kw.startswith("-") - ) - return False - - def bad_split_check(self, **kwargs): - """Checks for bad category/package splits. - - @param pkgdir: string: path - @returns: dictionary - """ - pkgdir = kwargs.get("pkgdir") - myesplit = portage.pkgsplit(self.y_ebuild) - is_bad_split = myesplit is None or myesplit[0] != self.xpkg.split("/")[-1] - if is_bad_split: - is_pv_toolong = pv_toolong_re.search(myesplit[1]) - is_pv_toolong2 = pv_toolong_re.search(myesplit[2]) - if is_pv_toolong or is_pv_toolong2: - self.qatracker.add_error( - "ebuild.invalidname", self.xpkg + "/" + self.y_ebuild + ".ebuild" - ) - return True - elif myesplit[0] != pkgdir: - print(pkgdir, myesplit[0]) - self.qatracker.add_error( - "ebuild.namenomatch", self.xpkg + "/" + self.y_ebuild + ".ebuild" - ) - return True - return False - - def pkg_invalid(self, **kwargs): - """Sets some pkg info and checks for invalid packages - - @param validity_future: Future instance - @returns: dictionary, including {pkg object} - """ - fuse = kwargs.get("validity_future") - dyn_pkg = kwargs.get("pkg") - if self.pkg.invalid: - for k, msgs in self.pkg.invalid.items(): - for msg in msgs: - self.qatracker.add_error(k, "%s: %s" % (self.relative_path, msg)) - # update the dynamic data - fuse.set(False, ignore_InvalidState=True) - dyn_pkg.set(self.pkg) - return True - # update the dynamic data - dyn_pkg.set(self.pkg) - return False - - def check_isebuild(self, **kwargs): - """Test the file for qualifications that is is an ebuild - - @param checkdirlist: list of files in the current package directory - @param checkdir: current package directory path - @param xpkg: current package directory being checked - @param validity_future: Future instance - @returns: dictionary, including {pkgs, can_force} - """ - checkdirlist = kwargs.get("checkdirlist").get() - checkdir = kwargs.get("checkdir") - xpkg = kwargs.get("xpkg") - fuse = kwargs.get("validity_future") - can_force = kwargs.get("can_force") - self.continue_ = False - ebuildlist = [] - pkgs = {} - for y in checkdirlist: - file_is_ebuild = y.endswith(".ebuild") - file_should_be_non_executable = ( - y in self.repo_settings.qadata.no_exec or file_is_ebuild - ) - - if file_should_be_non_executable: - file_is_executable = ( - stat.S_IMODE(os.stat(os.path.join(checkdir, y)).st_mode) & 0o111 - ) - - if file_is_executable: - self.qatracker.add_error( - "file.executable", os.path.join(checkdir, y) - ) - if file_is_ebuild: - pf = y[:-7] - ebuildlist.append(pf) - catdir = xpkg.split("/")[0] - cpv = "%s/%s" % (catdir, pf) - allvars = self.repo_settings.qadata.allvars - try: - myaux = dict(zip(allvars, self.portdb.aux_get(cpv, allvars))) - except KeyError: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("ebuild.syntax", os.path.join(xpkg, y)) - continue - except IOError: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("ebuild.output", os.path.join(xpkg, y)) - continue - except InvalidPackageName: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error( - "ebuild.invalidname", os.path.join(xpkg, y) - ) - continue - if not portage.eapi_is_supported(myaux["EAPI"]): - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("EAPI.unsupported", os.path.join(xpkg, y)) - continue - pkgs[pf] = Package( - cpv=cpv, - metadata=myaux, - root_config=self.root_config, - type_name="ebuild", - ) - - if len(pkgs) != len(ebuildlist): - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - self.continue_ = True - can_force.set(False, ignore_InvalidState=True) - self.pkgs = pkgs - # set our updated data - dyn_pkgs = kwargs.get("pkgs") - dyn_pkgs.set(pkgs) - return self.continue_ - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.check_isebuild]) - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return ( - True, - [self.check, self.set_pkg_data, self.bad_split_check, self.pkg_invalid], - ) diff --git a/repoman/lib/repoman/modules/scan/ebuild/multicheck.py b/repoman/lib/repoman/modules/scan/ebuild/multicheck.py deleted file mode 100644 index 1db0dfc92a9..00000000000 --- a/repoman/lib/repoman/modules/scan/ebuild/multicheck.py +++ /dev/null @@ -1,62 +0,0 @@ -"""multicheck.py -Perform multiple different checks on an ebuild -""" - -import io - -from portage import _encodings, _unicode_encode - -from repoman.modules.scan.scanbase import ScanBase -from repoman.modules.linechecks.controller import LineCheckController - - -class MultiCheck(ScanBase): - """Class to run multiple different checks on an ebuild""" - - def __init__(self, **kwargs): - """Class init - - @param qatracker: QATracker instance - @param options: the run time cli options - """ - self.qatracker = kwargs.get("qatracker") - self.options = kwargs.get("options") - self.controller = LineCheckController( - kwargs.get("repo_settings"), kwargs.get("linechecks") - ) - self.controller.checks_init(self.options.experimental_inherit == "y") - - def check(self, **kwargs): - """Check the ebuild for utf-8 encoding - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - """ - ebuild = kwargs.get("ebuild").get() - pkg = kwargs.get("pkg").get() - try: - # All ebuilds should have utf_8 encoding. - f = io.open( - _unicode_encode( - ebuild.full_path, encoding=_encodings["fs"], errors="strict" - ), - mode="r", - encoding=_encodings["repo.content"], - ) - try: - for check_name, e in self.controller.run_checks(f, pkg): - self.qatracker.add_error( - check_name, ebuild.relative_path + ": %s" % e - ) - finally: - f.close() - except UnicodeDecodeError: - # A file.UTF8 failure will have already been recorded. - pass - return False - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/eclasses/__init__.py b/repoman/lib/repoman/modules/scan/eclasses/__init__.py deleted file mode 100644 index 8a14000e8fb..00000000000 --- a/repoman/lib/repoman/modules/scan/eclasses/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eclasses plug-in module for repoman. -Performs an live and ruby eclass checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "eclasses", - "description": doc, - "provides": { - "live-module": { - "name": "live", - "sourcefile": "live", - "class": "LiveEclassChecks", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": [ - "qatracker", - "repo_metadata", - "repo_settings", - ], - "func_kwargs": { - "ebuild": (None, None), - "pkg": (None, None), - "xpkg": (None, None), - "y_ebuild": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - "ruby-module": { - "name": "ruby", - "sourcefile": "ruby", - "class": "RubyEclassChecks", - "description": doc, - "functions": ["check"], - "func_kwargs": {}, - "mod_kwargs": ["qatracker", "repo_settings"], - "func_kwargs": { - "ebuild": (None, None), - "pkg": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/eclasses/live.py b/repoman/lib/repoman/modules/scan/eclasses/live.py deleted file mode 100644 index 82ee168ed32..00000000000 --- a/repoman/lib/repoman/modules/scan/eclasses/live.py +++ /dev/null @@ -1,77 +0,0 @@ -"""live.py -Performs Live eclass checks -""" - -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase - - -class LiveEclassChecks(ScanBase): - """Performs checks for the usage of Live eclasses in ebuilds""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - """ - self.qatracker = kwargs.get("qatracker") - self.pmaskdict = kwargs.get("repo_metadata")["pmaskdict"] - self.repo_settings = kwargs.get("repo_settings") - - def check(self, **kwargs): - """Ebuilds that inherit a "Live" eclass (darcs, subversion, git, cvs, - etc..) should not be allowed to be marked stable - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: boolean - """ - pkg = kwargs.get("pkg").result() - package = kwargs.get("xpkg") - ebuild = kwargs.get("ebuild").get() - y_ebuild = kwargs.get("y_ebuild") - - if ebuild.live_ebuild and self.repo_settings.repo_config.name == "gentoo": - return self.check_live(pkg, package, ebuild, y_ebuild) - return False - - def check_live(self, pkg, package, ebuild, y_ebuild): - """Perform the live vcs check - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: boolean - """ - keywords = ebuild.keywords - is_stable = lambda kw: not kw.startswith("~") and not kw.startswith("-") - bad_stable_keywords = list(filter(is_stable, keywords)) - - if bad_stable_keywords: - self.qatracker.add_error( - "LIVEVCS.stable", - "%s/%s.ebuild with stable keywords: %s" - % (package, y_ebuild, bad_stable_keywords), - ) - - good_keywords_exist = len(bad_stable_keywords) < len(keywords) - if good_keywords_exist and not self._has_global_mask(pkg, self.pmaskdict): - self.qatracker.add_error("LIVEVCS.unmasked", ebuild.relative_path) - return False - - @staticmethod - def _has_global_mask(pkg, global_pmaskdict): - mask_atoms = global_pmaskdict.get(pkg.cp) - if mask_atoms: - pkg_list = [pkg] - for x in mask_atoms: - if portage.dep.match_from_list(x, pkg_list): - return x - return None - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/eclasses/ruby.py b/repoman/lib/repoman/modules/scan/eclasses/ruby.py deleted file mode 100644 index 16043ee0c88..00000000000 --- a/repoman/lib/repoman/modules/scan/eclasses/ruby.py +++ /dev/null @@ -1,49 +0,0 @@ -"""ruby.py -Performs Ruby eclass checks -""" - -from repoman.modules.scan.scanbase import ScanBase - - -class RubyEclassChecks(ScanBase): - """Performs checks for the usage of Ruby eclasses in ebuilds""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - """ - super(RubyEclassChecks, self).__init__(**kwargs) - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - self.old_ruby_eclasses = ["ruby-ng", "ruby-fakegem", "ruby"] - - def check(self, **kwargs): - """Check ebuilds that inherit the ruby eclasses - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - """ - pkg = kwargs.get("pkg").get() - ebuild = kwargs.get("ebuild").get() - is_inherited = lambda eclass: eclass in pkg.inherited - is_old_ruby_eclass_inherited = filter(is_inherited, self.old_ruby_eclasses) - - if is_old_ruby_eclass_inherited: - ruby_intersection = pkg.iuse.all.intersection( - self.repo_settings.qadata.ruby_deprecated - ) - - if ruby_intersection: - for myruby in ruby_intersection: - self.qatracker.add_error( - "IUSE.rubydeprecated", - (ebuild.relative_path + ": Deprecated ruby target: %s") - % myruby, - ) - return False - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/fetch/__init__.py b/repoman/lib/repoman/modules/scan/fetch/__init__.py deleted file mode 100644 index 9bd9d468d48..00000000000 --- a/repoman/lib/repoman/modules/scan/fetch/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """fetches plug-in module for repoman. -Performs fetch related checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "fetches", - "description": doc, - "provides": { - "fetches-module": { - "name": "fetches", - "sourcefile": "fetches", - "class": "FetchChecks", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": [ - "portdb", - "qatracker", - "repo_settings", - "vcs_settings", - ], - "func_kwargs": { - "changed": (None, None), - "checkdir": (None, None), - "checkdir_relative": (None, None), - "ebuild": (None, None), - "xpkg": (None, None), - }, - "module_runsIn": ["pkgs", "ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/fetch/fetches.py b/repoman/lib/repoman/modules/scan/fetch/fetches.py deleted file mode 100644 index 56a36a13fab..00000000000 --- a/repoman/lib/repoman/modules/scan/fetch/fetches.py +++ /dev/null @@ -1,205 +0,0 @@ -# -*- coding:utf-8 -*- - -"""fetches.py -Performs the src_uri fetchlist and files checks -""" - -from stat import S_ISDIR - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.vcs.vcs import vcs_new_changed -from repoman.modules.scan.scanbase import ScanBase - -from portage import os - - -class FetchChecks(ScanBase): - """Performs checks on the files needed for the ebuild""" - - def __init__(self, **kwargs): - """ - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: repository settings instance - @param vcs_settings: VCSSettings instance - """ - super(FetchChecks, self).__init__(**kwargs) - self.portdb = kwargs.get("portdb") - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - self.repoman_settings = self.repo_settings.repoman_settings - self.vcs_settings = kwargs.get("vcs_settings") - self._src_uri_error = False - - # TODO: Build a regex instead here, for the SRC_URI.mirror check. - self.thirdpartymirrors = {} - profile_thirdpartymirrors = ( - self.repo_settings.repoman_settings.thirdpartymirrors().items() - ) - for mirror_alias, mirrors in profile_thirdpartymirrors: - # Skip thirdpartymirrors that do not list more than one mirror - # anymore. There is no point in using mirror:// there and this - # means that the thirdpartymirrors entry will most likely - # be removed anyway. - if len(mirrors) <= 1: - continue - for mirror in mirrors: - if not mirror.endswith("/"): - mirror += "/" - self.thirdpartymirrors[mirror] = mirror_alias - - def check(self, **kwargs): - """Checks the ebuild sources and files for errors - - @param xpkg: the pacakge being checked - @param checkdir: string, directory path - @param checkdir_relative: repolevel determined path - @returns: boolean - """ - xpkg = kwargs.get("xpkg") - checkdir = kwargs.get("checkdir") - checkdir_relative = kwargs.get("checkdir_relative") - changed = kwargs.get("changed").changed - new = kwargs.get("changed").new - _digests = self.digests(checkdir) - fetchlist_dict = portage.FetchlistDict( - checkdir, self.repoman_settings, self.portdb - ) - myfiles_all = [] - self._src_uri_error = False - for mykey in fetchlist_dict: - try: - myfiles_all.extend(fetchlist_dict[mykey]) - except portage.exception.InvalidDependString as e: - self._src_uri_error = True - try: - self.portdb.aux_get(mykey, ["SRC_URI"]) - except KeyError: - # This will be reported as an "ebuild.syntax" error. - pass - else: - self.qatracker.add_error( - "SRC_URI.syntax", "%s.ebuild SRC_URI: %s" % (mykey, e) - ) - del fetchlist_dict - if not self._src_uri_error: - # This test can produce false positives if SRC_URI could not - # be parsed for one or more ebuilds. There's no point in - # producing a false error here since the root cause will - # produce a valid error elsewhere, such as "SRC_URI.syntax" - # or "ebuild.sytax". - myfiles_all = set(myfiles_all) - for entry in _digests: - if entry not in myfiles_all: - self.qatracker.add_error("digest.unused", checkdir + "::" + entry) - for entry in myfiles_all: - if entry not in _digests: - self.qatracker.add_error("digest.missing", checkdir + "::" + entry) - del myfiles_all - - if os.path.exists(checkdir + "/files"): - filesdirlist = os.listdir(checkdir + "/files") - - # Recurse through files directory, use filesdirlist as a stack; - # appending directories as needed, - # so people can't hide > 20k files in a subdirectory. - while filesdirlist: - y = filesdirlist.pop(0) - relative_path = os.path.join(xpkg, "files", y) - full_path = os.path.join(self.repo_settings.repodir, relative_path) - try: - mystat = os.stat(full_path) - except OSError as oe: - if oe.errno == 2: - # don't worry about it. it likely was removed via fix above. - continue - else: - raise oe - if S_ISDIR(mystat.st_mode): - if self.vcs_settings.status.isVcsDir(y): - continue - for z in os.listdir(checkdir + "/files/" + y): - if self.vcs_settings.status.isVcsDir(z): - continue - filesdirlist.append(y + "/" + z) - # Current policy is no files over 20 KiB, these are the checks. - # File size over 20 KiB causes an error. - elif mystat.st_size > 20480: - self.qatracker.add_error( - "file.size", - "(%d KiB) %s/files/%s" % (mystat.st_size // 1024, xpkg, y), - ) - elif mystat.st_size == 0: - self.qatracker.add_error("file.empty", "%s/files/%s" % (xpkg, y)) - - index = self.repo_settings.repo_config.find_invalid_path_char(y) - if index != -1: - y_relative = os.path.join(checkdir_relative, "files", y) - if self.vcs_settings.vcs is not None and not vcs_new_changed( - y_relative, changed, new - ): - # If the file isn't in the VCS new or changed set, then - # assume that it's an irrelevant temporary file (Manifest - # entries are not generated for file names containing - # prohibited characters). See bug #406877. - index = -1 - if index != -1: - self.qatracker.add_error( - "file.name", "%s/files/%s: char '%s'" % (checkdir, y, y[index]) - ) - return False - - def digests(self, checkdir): - """Returns the freshly loaded digests - - @param checkdir: string, directory path - """ - mf = self.repoman_settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(checkdir)) - ) - mf = mf.load_manifest(checkdir, self.repoman_settings["DISTDIR"]) - _digests = mf.getTypeDigests("DIST") - del mf - return _digests - - def check_mirrors(self, **kwargs): - """Check that URIs don't reference a server from thirdpartymirrors - - @param ebuild: Ebuild which we check (object). - @returns: boolean - """ - ebuild = kwargs.get("ebuild").get() - - for uri in portage.dep.use_reduce( - ebuild.metadata["SRC_URI"], - matchall=True, - is_src_uri=True, - eapi=ebuild.eapi, - flat=True, - ): - contains_mirror = False - for mirror, mirror_alias in self.thirdpartymirrors.items(): - if uri.startswith(mirror): - contains_mirror = True - break - if not contains_mirror: - continue - - new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror) :]) - self.qatracker.add_error( - "SRC_URI.mirror", - "%s: '%s' found in thirdpartymirrors, use '%s'" - % (ebuild.relative_path, mirror, new_uri), - ) - return False - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.check]) - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check_mirrors]) diff --git a/repoman/lib/repoman/modules/scan/keywords/__init__.py b/repoman/lib/repoman/modules/scan/keywords/__init__.py deleted file mode 100644 index a830e35ba62..00000000000 --- a/repoman/lib/repoman/modules/scan/keywords/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Keywords plug-in module for repoman. -Performs keywords checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "keywords", - "description": doc, - "provides": { - "keywords-module": { - "name": "keywords", - "sourcefile": "keywords", - "class": "KeywordChecks", - "description": doc, - "functions": ["prepare", "check"], - "func_desc": {}, - "mod_kwargs": [ - "qatracker", - "options", - "repo_metadata", - "profiles", - ], - "func_kwargs": { - "changed": (None, None), - "ebuild": ("Future", "UNSET"), - "pkg": ("Future", "UNSET"), - "xpkg": None, - "y_ebuild": (None, None), - }, - "module_runsIn": ["pkgs", "ebuilds", "final"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/keywords/keywords.py b/repoman/lib/repoman/modules/scan/keywords/keywords.py deleted file mode 100644 index 1e00421c6c7..00000000000 --- a/repoman/lib/repoman/modules/scan/keywords/keywords.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- coding:utf-8 -*- - -"""keywords.py -Perform KEYWORDS related checks - -""" - -from repoman.modules.scan.scanbase import ScanBase - - -class KeywordChecks(ScanBase): - """Perform checks on the KEYWORDS of an ebuild""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - @param options: argparse options instance - """ - super(KeywordChecks, self).__init__(**kwargs) - self.qatracker = kwargs.get("qatracker") - self.options = kwargs.get("options") - self.repo_metadata = kwargs.get("repo_metadata") - self.profiles = kwargs.get("profiles") - self.slot_keywords = {} - - def prepare(self, **kwargs): - """Prepare the checks for the next package.""" - self.slot_keywords = {} - self.dropped_keywords = {} - return False - - def check(self, **kwargs): - """Perform the check. - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @param ebuild_archs: Just the architectures (no prefixes) of the ebuild. - @param changed: Changes instance - @returns: dictionary - """ - pkg = kwargs.get("pkg").get() - xpkg = kwargs.get("xpkg") - ebuild = kwargs.get("ebuild").get() - y_ebuild = kwargs.get("y_ebuild") - changed = kwargs.get("changed") - if not self.options.straight_to_stable: - self._checkAddedWithStableKeywords( - xpkg, ebuild, y_ebuild, ebuild.keywords, changed - ) - - self._checkForDroppedKeywords(pkg, ebuild, ebuild.archs) - - self._checkForInvalidKeywords(ebuild, xpkg, y_ebuild) - - self._checkForMaskLikeKeywords(xpkg, y_ebuild, ebuild.keywords) - - self._checkForUnsortedKeywords(ebuild, xpkg, y_ebuild) - - self.slot_keywords[pkg.slot].update(ebuild.archs) - return False - - def check_dropped_keywords(self, **kwargs): - """Report on any dropped keywords for the latest ebuild in a slot - - @returns: boolean - """ - for ebuild, arches in self.dropped_keywords.values(): - if arches: - self.qatracker.add_error( - "KEYWORDS.dropped", "%s: %s" % (ebuild, " ".join(sorted(arches))) - ) - return False - - @staticmethod - def _isKeywordStable(keyword): - return not keyword.startswith("~") and not keyword.startswith("-") - - def _checkAddedWithStableKeywords( - self, package, ebuild, y_ebuild, keywords, changed - ): - catdir, pkgdir = package.split("/") - - stable_keywords = list(filter(self._isKeywordStable, keywords)) - if stable_keywords: - if ebuild.ebuild_path in changed.new_ebuilds and catdir != "virtual": - stable_keywords.sort() - self.qatracker.add_error( - "KEYWORDS.stable", - "%s/%s.ebuild added with stable keywords: %s" - % (package, y_ebuild, " ".join(stable_keywords)), - ) - - def _checkForDroppedKeywords(self, pkg, ebuild, ebuild_archs): - previous_keywords = self.slot_keywords.get(pkg.slot) - if previous_keywords is None: - self.slot_keywords[pkg.slot] = set() - elif ebuild_archs and "*" not in ebuild_archs and not ebuild.live_ebuild: - self.slot_keywords[pkg.slot].update(ebuild_archs) - dropped_keywords = previous_keywords.difference(ebuild_archs) - self.dropped_keywords[pkg.slot] = ( - ebuild.relative_path, - {arch for arch in dropped_keywords}, - ) - - def _checkForInvalidKeywords(self, ebuild, xpkg, y_ebuild): - myuse = ebuild.keywords - - for mykey in myuse: - if mykey not in ("-*", "*", "~*"): - myskey = mykey - - if not self._isKeywordStable(myskey[:1]): - myskey = myskey[1:] - - if myskey not in self.repo_metadata["kwlist"]: - self.qatracker.add_error( - "KEYWORDS.invalid", "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mykey) - ) - elif myskey not in self.profiles: - self.qatracker.add_error( - "KEYWORDS.invalid", - "%s/%s.ebuild: %s (profile invalid)" % (xpkg, y_ebuild, mykey), - ) - - def _checkForMaskLikeKeywords(self, xpkg, y_ebuild, keywords): - # KEYWORDS="-*" is a stupid replacement for package.mask - # and screws general KEYWORDS semantics - if "-*" in keywords: - haskeyword = False - - for kw in keywords: - if kw[0] == "~": - kw = kw[1:] - if kw in self.repo_metadata["kwlist"]: - haskeyword = True - - if not haskeyword: - self.qatracker.add_error( - "KEYWORDS.stupid", "%s/%s.ebuild" % (xpkg, y_ebuild) - ) - - def _checkForUnsortedKeywords(self, ebuild, xpkg, y_ebuild): - """Ebuilds that contain KEYWORDS - which are not sorted alphabetically.""" - - def sort_keywords(kw): - # Split keywords containing hyphens. The part after - # the hyphen should be treated as the primary key. - # This is consistent with ekeyword. - parts = list(reversed(kw.lstrip("~-").split("-", 1))) - # Hack to make sure that keywords - # without hyphens are sorted first - if len(parts) == 1: - parts.insert(0, "") - return parts - - sorted_keywords = sorted(ebuild.keywords, key=sort_keywords) - if sorted_keywords != ebuild.keywords: - self.qatracker.add_error( - "KEYWORDS.unsorted", - "%s/%s.ebuild contains unsorted keywords" % (xpkg, y_ebuild), - ) - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.prepare]) - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.check]) - - @property - def runInFinal(self): - """Final package level scans""" - return (True, [self.check_dropped_keywords]) diff --git a/repoman/lib/repoman/modules/scan/manifest/__init__.py b/repoman/lib/repoman/modules/scan/manifest/__init__.py deleted file mode 100644 index 2315a2a7e6c..00000000000 --- a/repoman/lib/repoman/modules/scan/manifest/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Ebuild plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "manifest", - "description": doc, - "provides": { - "manifest-module": { - "name": "manifests", - "sourcefile": "manifests", - "class": "Manifests", - "description": doc, - "functions": ["check", "create_manifest", "digest_check"], - "func_desc": {}, - "mod_kwargs": [ - "options", - "portdb", - "qatracker", - "repo_settings", - ], - "func_kwargs": { - "checkdir": (None, None), - "xpkg": (None, None), - }, - "module_runsIn": ["pkgs"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/manifest/manifests.py b/repoman/lib/repoman/modules/scan/manifest/manifests.py deleted file mode 100644 index 24b4e7f9b4b..00000000000 --- a/repoman/lib/repoman/modules/scan/manifest/manifests.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding:utf-8 -*- - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase - -from portage import os - - -class Manifests(ScanBase): - """Creates as well as checks pkg Manifest entries/files""" - - def __init__(self, **kwargs): - """Class init - - @param options: the run time cli options - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: repository settings instance - """ - self.options = kwargs.get("options") - self.portdb = kwargs.get("portdb") - self.qatracker = kwargs.get("qatracker") - self.repoman_settings = kwargs.get("repo_settings").repoman_settings - - def check(self, **kwargs): - """Perform a changelog and untracked checks on the ebuild - - @param xpkg: Package in which we check (object). - @param checkdir: the current package directory - @returns: dictionary - """ - checkdir = kwargs.get("checkdir") - xpkg = kwargs.get("xpkg") - if self.options.pretend: - return False - self.digest_check(xpkg, checkdir) - if self.options.mode == "manifest-check": - return True - return False - - def digest_check(self, xpkg, checkdir): - """Check the manifest entries, report any Q/A errors - - @param xpkg: the cat/pkg name to check - @param checkdir: the directory path to check""" - self.repoman_settings["O"] = checkdir - self.repoman_settings["PORTAGE_QUIET"] = "1" - if not portage.digestcheck([], self.repoman_settings, strict=1): - self.qatracker.add_error("manifest.bad", os.path.join(xpkg, "Manifest")) - self.repoman_settings.pop("PORTAGE_QUIET", None) - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/metadata/__init__.py b/repoman/lib/repoman/modules/scan/metadata/__init__.py deleted file mode 100644 index c133b1b37dd..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Metadata plug-in module for repoman. -Performs metadata checks on packages.""" -__doc__ = doc[:] - - -module_spec = { - "name": "metadata", - "description": doc, - "provides": { - "pkg-metadata": { - "name": "pkgmetadata", - "sourcefile": "pkgmetadata", - "class": "PkgMetadata", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": [ - "repo_settings", - "qatracker", - "options", - "metadata_xsd", - "uselist", - ], - "func_kwargs": { - "checkdir": (None, None), - "checkdirlist": (None, None), - "ebuild": (None, None), - "pkg": (None, None), - "repolevel": (None, None), - "validity_future": (None, None), - "xpkg": (None, None), - "y_ebuild": (None, None), - }, - "module_runsIn": ["pkgs", "ebuilds", "final"], - }, - "ebuild-metadata": { - "name": "ebuild_metadata", - "sourcefile": "ebuild_metadata", - "class": "EbuildMetadata", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": [ - "qatracker", - "repo_settings", - ], - "func_kwargs": { - "catdir": (None, None), - "ebuild": (None, None), - "xpkg": (None, None), - "y_ebuild": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - "description-metadata": { - "name": "description", - "sourcefile": "description", - "class": "DescriptionChecks", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": ["qatracker", "repo_settings"], - "func_kwargs": { - "ebuild": (None, None), - "pkg": ("Future", "UNSET"), - }, - "module_runsIn": ["ebuilds"], - }, - "restrict-metadata": { - "name": "restrict", - "sourcefile": "restrict", - "class": "RestrictChecks", - "description": doc, - "functions": ["check"], - "func_desc": {}, - "mod_kwargs": ["qatracker", "repo_settings"], - "func_kwargs": { - "ebuild": (None, None), - "xpkg": (None, None), - "y_ebuild": (None, None), - }, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/metadata/description.py b/repoman/lib/repoman/modules/scan/metadata/description.py deleted file mode 100644 index c9fb5e7f9ba..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/description.py +++ /dev/null @@ -1,44 +0,0 @@ -"""description.py -Perform checks on the DESCRIPTION variable. -""" - -from repoman.modules.scan.scanbase import ScanBase - - -class DescriptionChecks(ScanBase): - """Perform checks on the DESCRIPTION variable.""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - """ - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - - def checkTooLong(self, **kwargs): - """ - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - """ - ebuild = kwargs.get("ebuild").get() - pkg = kwargs.get("pkg").get() - # 14 is the length of DESCRIPTION="" - if len(pkg._metadata["DESCRIPTION"]) > self.repo_settings.qadata.max_desc_len: - self.qatracker.add_error( - "DESCRIPTION.toolong", - "%s: DESCRIPTION is %d characters (max %d)" - % ( - ebuild.relative_path, - len(pkg._metadata["DESCRIPTION"]), - self.repo_settings.qadata.max_desc_len, - ), - ) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (True, [self.checkTooLong]) diff --git a/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py b/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py deleted file mode 100644 index fd0552ce72a..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- coding:utf-8 -*- - -"""Ebuild Metadata Checks""" - -import re - -from repoman.modules.scan.scanbase import ScanBase - -from portage.dep import use_reduce - -NON_ASCII_RE = re.compile(r"[^\x00-\x7f]") -URISCHEME_RE = re.compile(r"^[a-z][0-9a-z\-\.\+]+://") - - -class EbuildMetadata(ScanBase): - def __init__(self, **kwargs): - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - - def invalidchar(self, **kwargs): - ebuild = kwargs.get("ebuild").get() - for k, v in ebuild.metadata.items(): - if not isinstance(v, str): - continue - m = NON_ASCII_RE.search(v) - if m is not None: - self.qatracker.add_error( - "variable.invalidchar", - "%s: %s variable contains non-ASCII " - "character at position %s" - % (ebuild.relative_path, k, m.start() + 1), - ) - return False - - def missing(self, **kwargs): - ebuild = kwargs.get("ebuild").get() - for pos, missing_var in enumerate(self.repo_settings.qadata.missingvars): - if not ebuild.metadata.get(missing_var): - if kwargs.get("catdir") in ( - "acct-group", - "acct-user", - "virtual", - ) and missing_var in ("HOMEPAGE", "LICENSE"): - continue - if ebuild.live_ebuild and missing_var == "KEYWORDS": - continue - myqakey = self.repo_settings.qadata.missingvars[pos] + ".missing" - self.qatracker.add_error( - myqakey, - "%s/%s.ebuild" % (kwargs.get("xpkg"), kwargs.get("y_ebuild")), - ) - return False - - def virtual(self, **kwargs): - ebuild = kwargs.get("ebuild").get() - if kwargs.get("catdir") == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if ebuild.metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, ebuild.relative_path) - return False - - def homepage_urischeme(self, **kwargs): - ebuild = kwargs.get("ebuild").get() - if kwargs.get("catdir") != "virtual": - for homepage in use_reduce( - ebuild.metadata["HOMEPAGE"], matchall=True, flat=True - ): - if URISCHEME_RE.match(homepage) is None: - self.qatracker.add_error( - "HOMEPAGE.missingurischeme", ebuild.relative_path - ) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return ( - True, - [self.invalidchar, self.missing, self.virtual, self.homepage_urischeme], - ) diff --git a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py b/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py deleted file mode 100644 index 0fb97a0dfc0..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/pkgmetadata.py +++ /dev/null @@ -1,221 +0,0 @@ -# -*- coding:utf-8 -*- - -"""Package Metadata Checks operations""" - -import sys -import re - -from itertools import chain -from collections import Counter - -try: - from lxml import etree - from lxml.etree import ParserError -except (SystemExit, KeyboardInterrupt): - raise -except (ImportError, SystemError, RuntimeError, Exception): - # broken or missing xml support - # https://bugs.python.org/issue14988 - msg = ["Please emerge dev-python/lxml in order to use repoman."] - from portage.output import EOutput - - out = EOutput() - for line in msg: - out.eerror(line) - sys.exit(1) - -# import our initialized portage instance -from repoman._portage import portage # pylint: disable=unused-import -from repoman.metadata import metadata_dtd_uri -from repoman.modules.scan.scanbase import ScanBase - -# pylint: disable=ungrouped-imports -from portage.exception import InvalidAtom -from portage import os -from portage.dep import Atom -from portage.xml.metadata import parse_metadata_use - -from .use_flags import USEFlagChecks - -metadata_xml_encoding = "UTF-8" -metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' % ( - metadata_xml_encoding, -) -metadata_doctype_name = "pkgmetadata" - - -class PkgMetadata(ScanBase, USEFlagChecks): - """Package metadata.xml checks""" - - def __init__(self, **kwargs): - """PkgMetadata init function - - @param repo_settings: settings instance - @param qatracker: QATracker instance - @param options: argparse options instance - @param metadata_xsd: path of metadata.xsd - """ - super(PkgMetadata, self).__init__(**kwargs) - repo_settings = kwargs.get("repo_settings") - self.qatracker = kwargs.get("qatracker") - self.options = kwargs.get("options") - self.metadata_xsd = kwargs.get("metadata_xsd") - self.globalUseFlags = kwargs.get("uselist") - self.repoman_settings = repo_settings.repoman_settings - self.musedict = {} - self.muselist = set() - - def check(self, **kwargs): - """Performs the checks on the metadata.xml for the package - @param xpkg: the pacakge being checked - @param checkdir: string, directory path - @param checkdirlist: list of checkdir's - @param repolevel: integer - @returns: boolean - """ - xpkg = kwargs.get("xpkg") - checkdir = kwargs.get("checkdir") - checkdirlist = kwargs.get("checkdirlist").get() - - self.musedict = {} - if self.options.mode in ["manifest"]: - self.muselist = frozenset(self.musedict) - return False - - # metadata.xml file check - if "metadata.xml" not in checkdirlist: - self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") - self.muselist = frozenset(self.musedict) - return False - - # metadata.xml parse check - metadata_bad = False - - # read metadata.xml into memory - try: - _metadata_xml = etree.parse(os.path.join(checkdir, "metadata.xml")) - except (ParserError, SyntaxError, EnvironmentError) as e: - metadata_bad = True - self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - self.muselist = frozenset(self.musedict) - return False - - indentation_chars = Counter() - for l in etree.tostring(_metadata_xml).splitlines(): - indentation_chars.update(re.match(rb"\s*", l).group(0)) - if len(indentation_chars) > 1: - self.qatracker.add_error( - "metadata.warning", - "%s/metadata.xml: %s" - % (xpkg, "inconsistent use of tabs and spaces in indentation"), - ) - - xml_encoding = _metadata_xml.docinfo.encoding - if xml_encoding.upper() != metadata_xml_encoding: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: " - "xml declaration encoding should be '%s', not '%s'" - % (xpkg, metadata_xml_encoding, xml_encoding), - ) - - if not _metadata_xml.docinfo.doctype: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing") - ) - else: - doctype_system = _metadata_xml.docinfo.system_url - if doctype_system != metadata_dtd_uri: - if doctype_system is None: - system_problem = "but it is undefined" - else: - system_problem = "not '%s'" % doctype_system - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: " - "DOCTYPE: SYSTEM should refer to '%s', %s" - % (xpkg, metadata_dtd_uri, system_problem), - ) - doctype_name = _metadata_xml.docinfo.doctype.split(" ")[1] - if doctype_name != metadata_doctype_name: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: " - "DOCTYPE: name should be '%s', not '%s'" - % (xpkg, metadata_doctype_name, doctype_name), - ) - - # load USE flags from metadata.xml - self.musedict = parse_metadata_use(_metadata_xml) - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: Invalid atom: %s" % (xpkg, e) - ) - else: - if atom.cp != xpkg: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: Atom contains " - "unexpected cat/pn: %s" % (xpkg, atom), - ) - - # Only carry out if in package directory or check forced - if not metadata_bad: - validator = etree.XMLSchema(file=self.metadata_xsd) - if not validator.validate(_metadata_xml): - self._add_validate_errors(xpkg, validator.error_log) - self.muselist = frozenset(self.musedict) - return False - - def check_unused(self, **kwargs): - """Reports on any unused metadata.xml use descriptions - - @param xpkg: the pacakge being checked - @param used_useflags: use flag list - @param validity_future: Future instance - """ - xpkg = kwargs.get("xpkg") - valid_state = kwargs.get("validity_future").get() - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if valid_state: - for myflag in self.muselist.difference(self.usedUseFlags): - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag), - ) - return False - - def _add_validate_errors(self, xpkg, log): - listed = set() - for error in log: - msg_prefix = error.message.split(":", 1)[0] - info = "%s %s" % (error.line, msg_prefix) - if info not in listed: - listed.add(info) - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" % (xpkg, error.line, error.message), - ) - - @property - def runInPkgs(self): - """Package level scans""" - return (True, [self.check]) - - @property - def runInEbuilds(self): - return (True, [self.check_useflags]) - - @property - def runInFinal(self): - """Final scans at the package level""" - return (True, [self.check_unused]) diff --git a/repoman/lib/repoman/modules/scan/metadata/restrict.py b/repoman/lib/repoman/modules/scan/metadata/restrict.py deleted file mode 100644 index 036af88f6e7..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/restrict.py +++ /dev/null @@ -1,58 +0,0 @@ -"""restrict.py -Perform checks on the RESTRICT variable. -""" - -# import our initialized portage instance -from repoman._portage import portage - -from repoman.modules.scan.scanbase import ScanBase - - -class RestrictChecks(ScanBase): - """Perform checks on the RESTRICT variable.""" - - def __init__(self, **kwargs): - """ - @param qatracker: QATracker instance - """ - self.qatracker = kwargs.get("qatracker") - self.repo_settings = kwargs.get("repo_settings") - if self.repo_settings.repo_config.restrict_allowed is None: - self._restrict_allowed = self.repo_settings.qadata.valid_restrict - else: - self._restrict_allowed = self.repo_settings.repo_config.restrict_allowed - - def check(self, **kwargs): - xpkg = kwargs.get("xpkg") - ebuild = kwargs.get("ebuild").get() - y_ebuild = kwargs.get("y_ebuild") - myrestrict = None - - try: - myrestrict = portage.dep.use_reduce( - ebuild.metadata["RESTRICT"], matchall=1, flat=True - ) - except portage.exception.InvalidDependString as e: - self.qatracker.add_error( - "RESTRICT.syntax", "%s: RESTRICT: %s" % (ebuild.relative_path, e) - ) - del e - - if myrestrict: - myrestrict = set(myrestrict) - mybadrestrict = myrestrict.difference(self._restrict_allowed) - - if mybadrestrict: - for mybad in mybadrestrict: - self.qatracker.add_error( - "RESTRICT.invalid", "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad) - ) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (True, [self.check]) diff --git a/repoman/lib/repoman/modules/scan/metadata/use_flags.py b/repoman/lib/repoman/modules/scan/metadata/use_flags.py deleted file mode 100644 index f2116800e16..00000000000 --- a/repoman/lib/repoman/modules/scan/metadata/use_flags.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding:utf-8 -*- - -"""use_flags.py -Performs USE flag related checks -""" - -# import our centrally initialized portage instance -from repoman._portage import portage - -from portage import eapi -from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use - - -class USEFlagChecks: - """Performs checks on USE flags listed in the ebuilds and metadata.xml""" - - def __init__(self, **kwargs): - """Class init - - @param qatracker: QATracker instance - @param globalUseFlags: Global USE flags - """ - super(USEFlagChecks, self).__init__() - self.qatracker = None - self.globalUseFlags = None - self.useFlags = [] - self.defaultUseFlags = [] - self.usedUseFlags = set() - - def check_useflags(self, **kwargs): - """Perform the check. - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: dictionary, including {ebuild_UsedUseFlags, used_useflags} - """ - pkg = kwargs.get("pkg").get() - package = kwargs.get("xpkg") - ebuild = kwargs.get("ebuild").get() - y_ebuild = kwargs.get("y_ebuild") - # reset state variables for the run - self.useFlags = [] - self.defaultUseFlags = [] - # perform the checks - self._checkGlobal(pkg) - self._checkMetadata(package, ebuild, y_ebuild, self.muselist) - self._checkRequiredUSE(pkg, ebuild) - return False - - def _checkGlobal(self, pkg): - for myflag in pkg._metadata["IUSE"].split(): - flag_name = myflag.lstrip("+-") - self.usedUseFlags.add(flag_name) - if myflag != flag_name: - self.defaultUseFlags.append(myflag) - if flag_name not in self.globalUseFlags: - self.useFlags.append(flag_name) - - def _checkMetadata(self, package, ebuild, y_ebuild, localUseFlags): - for mypos in range(len(self.useFlags) - 1, -1, -1): - if self.useFlags[mypos] and (self.useFlags[mypos] in localUseFlags): - del self.useFlags[mypos] - - if self.defaultUseFlags and not eapi_has_iuse_defaults(eapi): - for myflag in self.defaultUseFlags: - self.qatracker.add_error( - "EAPI.incompatible", - "%s: IUSE defaults" - " not supported with EAPI='%s': '%s'" - % (ebuild.relative_path, eapi, myflag), - ) - - for mypos in range(len(self.useFlags)): - self.qatracker.add_error( - "IUSE.invalid", - "%s/%s.ebuild: %s" % (package, y_ebuild, self.useFlags[mypos]), - ) - - def _checkRequiredUSE(self, pkg, ebuild): - required_use = pkg._metadata["REQUIRED_USE"] - if required_use: - if not eapi_has_required_use(eapi): - self.qatracker.add_error( - "EAPI.incompatible", - "%s: REQUIRED_USE" - " not supported with EAPI='%s'" - % ( - ebuild.relative_path, - eapi, - ), - ) - try: - portage.dep.check_required_use( - required_use, (), pkg.iuse.is_valid_flag, eapi=eapi - ) - except portage.exception.InvalidDependString as e: - self.qatracker.add_error( - "REQUIRED_USE.syntax", - "%s: REQUIRED_USE: %s" % (ebuild.relative_path, e), - ) - del e diff --git a/repoman/lib/repoman/modules/scan/module.py b/repoman/lib/repoman/modules/scan/module.py deleted file mode 100644 index 9e67f9bd618..00000000000 --- a/repoman/lib/repoman/modules/scan/module.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -moudules/scan/module.py -Module loading and run list generator -""" - -import logging -import os -import yaml - -import portage -from portage.module import InvalidModuleName, Modules -from portage.util import stack_lists -from repoman import _not_installed -from repoman.config import ConfigError - -MODULES_PATH = os.path.dirname(__file__) -# initial development debug info -logging.debug("module path: %s", MODULES_PATH) - - -class ModuleConfig: - """Holds the scan modules configuration information and - creates the ordered list of modulles to run""" - - def __init__(self, configpaths, valid_versions=None, repository_modules=False): - """Module init - - @param configpaths: ordered list of filepaths to load - """ - if repository_modules: - self.configpaths = [ - os.path.join(path, "repository.yaml") for path in configpaths - ] - elif _not_installed: - self.configpaths = [ - os.path.realpath( - os.path.join( - os.path.dirname( - os.path.dirname( - os.path.dirname( - os.path.dirname(os.path.dirname(__file__)) - ) - ) - ), - "cnf/repository/repository.yaml", - ) - ) - ] - else: - self.configpaths = [ - os.path.join( - portage.const.EPREFIX or "/", - "usr/share/repoman/repository/repository.yaml", - ) - ] - logging.debug("ModuleConfig; configpaths: %s", self.configpaths) - - self.controller = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") - logging.debug("ModuleConfig; module_names: %s", self.controller.module_names) - - self._configs = None - self.enabled = [] - self.pkgs_loop = [] - self.ebuilds_loop = [] - self.final_loop = [] - self.modules_forced = ["ebuild", "mtime"] - self.load_configs(valid_versions=valid_versions) - for loop in ["pkgs", "ebuilds", "final"]: - logging.debug("ModuleConfig; Processing loop %s", loop) - setattr(self, "%s_loop" % loop, self._determine_list(loop)) - self.linechecks = stack_lists( - c["linechecks_modules"].split() for c in self._configs - ) - - def load_configs(self, configpaths=None, valid_versions=None): - """load the config files in order - - @param configpaths: ordered list of filepaths to load - """ - if configpaths: - self.configpaths = configpaths - elif not self.configpaths: - logging.error("ModuleConfig; Error: No repository.yaml files defined") - configs = [] - for path in self.configpaths: - logging.debug("ModuleConfig; Processing: %s", path) - if os.path.exists(path): - try: - with open(path, "r") as inputfile: - configs.append(yaml.safe_load(inputfile)) - except IOError as error: - logging, error("Failed to load file: %s", inputfile) - logging.exception(error) - else: - if configs[-1]["version"] not in valid_versions: - raise ConfigError( - "Invalid file version: %s in: %s\nPlease upgrade repoman" - % (configs["version"], path) - ) - logging.debug("ModuleConfig; completed : %s", path) - logging.debug("ModuleConfig; new _configs: %s", configs) - self._configs = configs - - def _determine_list(self, loop): - """Determine the ordered list from the config data and - the moule_runsIn value in the module_spec - - @returns: list of modules - """ - lists = [c["scan_modules"].split() for c in self._configs] - stacked = self.modules_forced + stack_lists(lists) - mlist = [] - try: - for mod in stacked: - logging.debug( - "ModuleConfig; checking loop %s, module: %s, in: %s", - loop, - mod, - self.controller.get_spec(mod, "module_runsIn"), - ) - if loop in self.controller.get_spec(mod, "module_runsIn"): - mlist.append(mod) - except InvalidModuleName: - logging.error("ModuleConfig; unknown module: %s, skipping", mod) - - logging.debug("ModuleConfig; mlist: %s", mlist) - return mlist diff --git a/repoman/lib/repoman/modules/scan/options/__init__.py b/repoman/lib/repoman/modules/scan/options/__init__.py deleted file mode 100644 index ce57cbbfc62..00000000000 --- a/repoman/lib/repoman/modules/scan/options/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -doc = """Options plug-in module for repoman. -Performs option related actions on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - "name": "options", - "description": doc, - "provides": { - "options-module": { - "name": "options", - "sourcefile": "options", - "class": "Options", - "description": doc, - "functions": ["is_forced"], - "func_desc": {}, - "mod_kwargs": [ - "options", - ], - "func_kwargs": {}, - "module_runsIn": ["ebuilds"], - }, - }, - "version": 1, -} diff --git a/repoman/lib/repoman/modules/scan/options/options.py b/repoman/lib/repoman/modules/scan/options/options.py deleted file mode 100644 index 2a16be7ef2e..00000000000 --- a/repoman/lib/repoman/modules/scan/options/options.py +++ /dev/null @@ -1,27 +0,0 @@ -from repoman.modules.scan.scanbase import ScanBase - - -class Options(ScanBase): - def __init__(self, **kwargs): - """Class init function - - @param options: argparse options instance - """ - self.options = kwargs.get("options") - - def is_forced(self, **kwargs): - """Simple boolean function to trigger a skip past some additional checks - - @returns: dictionary - """ - if self.options.force: - # The dep_check() calls are the most expensive QA test. If --force - # is enabled, there's no point in wasting time on these since the - # user is intent on forcing the commit anyway. - return True - return False - - @property - def runInEbuilds(self): - """Ebuild level scans""" - return (True, [self.is_forced]) diff --git a/repoman/lib/repoman/modules/scan/scan.py b/repoman/lib/repoman/modules/scan/scan.py deleted file mode 100644 index 30de50c3577..00000000000 --- a/repoman/lib/repoman/modules/scan/scan.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding:utf-8 -*- - -""" -moudules/scan.py -Module specific package scan list generator -""" - -import logging -import os -import sys - -from repoman.errors import caterror - - -def scan(repolevel, reposplit, startdir, categories, repo_settings): - """Generate a list of pkgs to scan - - @param repolevel: integer, number of subdirectories deep from the tree root - @param reposplit: list of the path subdirs - @param startdir: the top level directory to begin scanning from - @param categories: list of known categories - @param repo_settings: repository settings instance - @returns: scanlist, sorted list of pkgs to scan - """ - scanlist = [] - if repolevel == 2: - # we are inside a category directory - catdir = reposplit[-1] - if catdir not in categories: - caterror(catdir, repo_settings.repodir) - mydirlist = os.listdir(startdir) - for x in mydirlist: - if x == "CVS" or x.startswith("."): - continue - if os.path.isdir(startdir + "/" + x): - scanlist.append(catdir + "/" + x) - # repo_subdir = catdir + os.sep - elif repolevel == 1: - for x in categories: - if not os.path.isdir(startdir + "/" + x): - continue - for y in os.listdir(startdir + "/" + x): - if y == "CVS" or y.startswith("."): - continue - if os.path.isdir(startdir + "/" + x + "/" + y): - scanlist.append(x + "/" + y) - # repo_subdir = "" - elif repolevel == 3: - catdir = reposplit[-2] - if catdir not in categories: - caterror(catdir, repo_settings.repodir) - scanlist.append(catdir + "/" + reposplit[-1]) - # repo_subdir = scanlist[-1] + os.sep - else: - msg = ( - "Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY" - + " from the current working directory" - ) - logging.critical(msg) - sys.exit(1) - - # repo_subdir_len = len(repo_subdir) - scanlist.sort() - - logging.debug("Found the following packages to scan:\n%s" % "\n".join(scanlist)) - - return scanlist diff --git a/repoman/lib/repoman/modules/scan/scanbase.py b/repoman/lib/repoman/modules/scan/scanbase.py deleted file mode 100644 index 5a7cd62193b..00000000000 --- a/repoman/lib/repoman/modules/scan/scanbase.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding:utf-8 -*- - - -class ScanBase: - """Skeleton class for performing a scan for one or more items - to check in a pkg directory or ebuild.""" - - def __init__(self, **kwargs): - """Class init - - @param kwargs: an optional dictionary of common repository - wide parameters that may be required. - """ - # Since no two checks are identicle as to what kwargs are needed, - # this does not define any from it here. - super(ScanBase, self).__init__() - - """ # sample check - def check_foo(self, **kwargs): - '''Class check skeleton function. Define this for a - specific check to perform. - - @param kwargs: an optional dictionary of dynamic package and or ebuild - specific data that may be required. Dynamic data can - vary depending what checks have run before it. - So execution order can be important. - ''' - # Insert the code for the check here - # It should return a dictionary of at least {'continue': False} - # The continue attribute will default to False if not returned. - # This will allow the loop to continue with the next check in the list. - # Include any additional dynamic data that needs to be added or updated. - return False # used as a continue True/False value - """ - - @property - def runInPkgs(self): - """Package level scans""" - # default no run (False) and empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_foo]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # next stage is runInEbuilds - return (False, []) - - @property - def runInEbuilds(self): - """Ebuild level scans""" - # default empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_bar]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # next stage is runInFinal - return (False, []) - - @property - def runInFinal(self): - """Final scans at the package level""" - # default empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_baz]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # runInFinal is currently the last stage of scans performed. - return (False, []) diff --git a/repoman/lib/repoman/modules/vcs/None/__init__.py b/repoman/lib/repoman/modules/vcs/None/__init__.py deleted file mode 100644 index dc9eb51ad84..00000000000 --- a/repoman/lib/repoman/modules/vcs/None/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """None (non vcs type) plug-in module for portage. -Performs various git actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "None", - "description": doc, - "provides": { - "None-module": { - "name": "None_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": False, - "needs_keyword_expansion": False, - }, - "None-changes": { - "name": "None_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/None/changes.py b/repoman/lib/repoman/modules/vcs/None/changes.py deleted file mode 100644 index 4164c84bf29..00000000000 --- a/repoman/lib/repoman/modules/vcs/None/changes.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -None module Changes class submodule -""" - -from repoman.modules.vcs.changes import ChangesBase - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "None" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - - def scan(self): - """VCS type scan function, looks for all detectable changes""" - pass - - def add_items(self, autoadd): - """Add files to the vcs's modified or new index - - @param autoadd: the files to add to the vcs modified index""" - pass - - def commit(self, myfiles, commitmessagefile): - """None commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - """ - commit_cmd = [] - # substitute a bogus vcs value for pretend output - commit_cmd.append("pretend") - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in myfiles) - - print("(%s)" % (" ".join(commit_cmd),)) - return 0 diff --git a/repoman/lib/repoman/modules/vcs/None/status.py b/repoman/lib/repoman/modules/vcs/None/status.py deleted file mode 100644 index c5809bc9e29..00000000000 --- a/repoman/lib/repoman/modules/vcs/None/status.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -None (non-VCS) module Status class submodule -""" - - -class Status: - """Performs status checks on the svn repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - return True - - @staticmethod - def detect_conflicts(options): - """Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - """ - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - return False - - @staticmethod - def isVcsDir(dirname): - """Is the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return False diff --git a/repoman/lib/repoman/modules/vcs/__init__.py b/repoman/lib/repoman/modules/vcs/__init__.py deleted file mode 100644 index 85cc979c059..00000000000 --- a/repoman/lib/repoman/modules/vcs/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -from portage.module import Modules - -path = os.path.dirname(__file__) -# initial development debug info -# print("module path:", path) - -module_controller = Modules(path=path, namepath="repoman.modules.vcs") - -# initial development debug info -# print(module_controller.module_names) -module_names = module_controller.module_names[:] diff --git a/repoman/lib/repoman/modules/vcs/bzr/__init__.py b/repoman/lib/repoman/modules/vcs/bzr/__init__.py deleted file mode 100644 index 982017da67d..00000000000 --- a/repoman/lib/repoman/modules/vcs/bzr/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Bazaar (bzr) plug-in module for portage. -Performs variaous Bazaar actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "bzr", - "description": doc, - "provides": { - "bzr-module": { - "name": "bzr_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": True, - "needs_keyword_expansion": False, - }, - "bzr-changes": { - "name": "bzr_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/bzr/changes.py b/repoman/lib/repoman/modules/vcs/bzr/changes.py deleted file mode 100644 index 0366b38621f..00000000000 --- a/repoman/lib/repoman/modules/vcs/bzr/changes.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Bazaar module Changes class submodule -""" - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._portage import portage # pylint: disable=unused-import -from portage import os -from portage.package.ebuild.digestgen import digestgen - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "bzr" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - """VCS type scan function, looks for all detectable changes""" - with repoman_popen("bzr status -S .") as f: - bzrstatus = f.readlines() - self.changed = [ - "./" + elem.split()[-1:][0].split("/")[-1:][0] - for elem in bzrstatus - if elem and elem[1:2] == "M" - ] - self.new = [ - "./" + elem.split()[-1:][0].split("/")[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] == "NK" or elem[0:1] == "R") - ] - self.removed = [ - "./" + elem.split()[-3:-2][0].split("/")[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] == "K" or elem[0:1] == "R") - ] - self.bzrstatus = bzrstatus - # Bazaar expands nothing. - - @property - def unadded(self): - """Bazzar method of getting the unadded files in the repository""" - if self._unadded is not None: - return self._unadded - self._unadded = [ - "./" + elem.rstrip().split()[1].split("/")[-1:][0] - for elem in self.bzrstatus - if elem.startswith("?") or elem[0:2] == " D" - ] - return self._unadded - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) diff --git a/repoman/lib/repoman/modules/vcs/bzr/status.py b/repoman/lib/repoman/modules/vcs/bzr/status.py deleted file mode 100644 index 5ca18c4f637..00000000000 --- a/repoman/lib/repoman/modules/vcs/bzr/status.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Bazaar module Status class submodule -""" - -from repoman._portage import portage -from portage import os - -# pylint: disable=ungrouped-imports -from repoman._subprocess import repoman_popen - - -class Status: - """Performs status checks on the svn repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - try: - myf = repoman_popen( - "bzr ls -v --kind=file " + portage._shell_quote(checkdir) - ) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[1:2] == "?": - continue - l = l.split()[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - return True - - @staticmethod - def detect_conflicts(options): - """Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - """ - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - return False - - @staticmethod - def isVcsDir(dirname): - """Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return dirname in [".bzr"] diff --git a/repoman/lib/repoman/modules/vcs/changes.py b/repoman/lib/repoman/modules/vcs/changes.py deleted file mode 100644 index c1c7b090965..00000000000 --- a/repoman/lib/repoman/modules/vcs/changes.py +++ /dev/null @@ -1,170 +0,0 @@ -""" -Base Changes class -""" - -import logging -import os -import subprocess -import sys -from itertools import chain - -from repoman._portage import portage -from portage import _unicode_encode -from portage.process import spawn - - -class ChangesBase: - """Base Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "None" - - def __init__(self, options, repo_settings): - """Class init function - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - self.options = options - self.repo_settings = repo_settings - self.repoman_settings = repo_settings.repoman_settings - self.vcs_settings = repo_settings.vcs_settings - self._reset() - - def _reset(self): - """Reset the class variables for a new run""" - self.new_ebuilds = set() - self.ebuilds = set() - self.changelogs = set() - self.changed = [] - self.new = [] - self.removed = [] - self.no_expansion = set() - self._expansion = None - self._deleted = None - self._unadded = None - - def scan(self): - """Scan the vcs for detectable changes. - - base method which calls the subclassing VCS module's _scan() - then updates some classwide variables. - """ - self._reset() - - if self.vcs: - self._scan() - self.new_ebuilds.update(x for x in self.new if x.endswith(".ebuild")) - self.ebuilds.update(x for x in self.changed if x.endswith(".ebuild")) - self.changelogs.update( - x - for x in chain(self.changed, self.new) - if os.path.basename(x) == "ChangeLog" - ) - - def _scan(self): - """Placeholder for subclassing""" - pass - - @property - def has_deleted(self): - """Placeholder for VCS that requires manual deletion of files""" - return self.deleted != [] - - @property - def has_changes(self): - """Placeholder for VCS repo common has changes result""" - changed = self.changed or self.new or self.removed or self.deleted - return changed != [] - - @property - def unadded(self): - """Override this function as needed""" - return [] - - @property - def deleted(self): - """Override this function as needed""" - return [] - - @property - def expansion(self): - """Override this function as needed""" - return {} - - def thick_manifest(self, updates, headers, no_expansion, expansion): - """Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - """ - pass - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - pass - - @staticmethod - def clear_attic(headers): - """Old CVS leftover - - @param headers: file headers""" - pass - - def update_index(self, mymanifests, myupdates): - """Update the vcs's modified index if it is needed - - @param mymanifests: manifest files updated - @param myupdates: other files updated""" - pass - - def add_items(self, autoadd): - """Add files to the vcs's modified or new index - - @param autoadd: the files to add to the vcs modified index""" - add_cmd = [self.vcs, "add"] - add_cmd += autoadd - if self.options.pretend: - portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd), noiselevel=-1) - else: - add_cmd = [_unicode_encode(arg) for arg in add_cmd] - retcode = subprocess.call(add_cmd) - if retcode != os.EX_OK: - logging.error( - "Exiting on %s error code: %s\n", self.vcs_settings.vcs, retcode - ) - sys.exit(retcode) - - def commit(self, commitfiles, commitmessagefile): - """Common generic commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - """ - commit_cmd = [] - commit_cmd.append(self.vcs) - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in commitfiles) - - if self.options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) - return 0 - else: - retval = spawn(commit_cmd, env=self.repo_settings.commit_env) - return retval diff --git a/repoman/lib/repoman/modules/vcs/cvs/__init__.py b/repoman/lib/repoman/modules/vcs/cvs/__init__.py deleted file mode 100644 index f71f806fb6a..00000000000 --- a/repoman/lib/repoman/modules/vcs/cvs/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """CVS (cvs) plug-in module for portage. -Performs variaous CVS actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "cvs", - "description": doc, - "provides": { - "cvs-status": { - "name": "cvs_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": True, - "needs_keyword_expansion": True, - }, - "cvs-changes": { - "name": "cvs_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/cvs/changes.py b/repoman/lib/repoman/modules/vcs/cvs/changes.py deleted file mode 100644 index be382f291d3..00000000000 --- a/repoman/lib/repoman/modules/vcs/cvs/changes.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -CVS module Changes class submodule -""" - -import re -from itertools import chain - -from repoman._portage import portage -from repoman.modules.vcs.changes import ChangesBase -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman._subprocess import repoman_getstatusoutput - -from portage import _encodings, _unicode_encode -from portage import cvstree, os -from portage.output import green -from portage.package.ebuild.digestgen import digestgen - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "cvs" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - self._tree = None - - def _scan(self): - """VCS type scan function, looks for all detectable changes""" - self._tree = portage.cvstree.getentries("./", recursive=1) - self.changed = cvstree.findchanged(self._tree, recursive=1, basedir="./") - self.new = cvstree.findnew(self._tree, recursive=1, basedir="./") - self.removed = cvstree.findremoved(self._tree, recursive=1, basedir="./") - bin_blob_pattern = re.compile("^-kb$") - self.no_expansion = set( - portage.cvstree.findoption( - self._tree, bin_blob_pattern, recursive=1, basedir="./" - ) - ) - - @property - def unadded(self): - """VCS method of getting the unadded files in the repository""" - if self._unadded is not None: - return self._unadded - self._unadded = portage.cvstree.findunadded( - self._tree, recursive=1, basedir="./" - ) - return self._unadded - - @staticmethod - def clear_attic(headers): - """Clear the attic (inactive files) - - @param headers: file headers - """ - cvs_header_re = re.compile(rb"^#\s*\$Header.*\$$") - attic_str = b"/Attic/" - attic_replace = b"/" - for x in headers: - f = open( - _unicode_encode(x, encoding=_encodings["fs"], errors="strict"), - mode="rb", - ) - mylines = f.readlines() - f.close() - modified = False - for i, line in enumerate(mylines): - if cvs_header_re.match(line) is not None and attic_str in line: - mylines[i] = line.replace(attic_str, attic_replace) - modified = True - if modified: - portage.util.write_atomic(x, b"".join(mylines), mode="wb") - - def thick_manifest(self, updates, headers, no_expansion, expansion): - """Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - """ - headerstring = r"'\$(Header|Id).*\$'" - - for _file in updates: - - # for CVS, no_expansion contains files that are excluded from expansion - if _file in no_expansion: - continue - - _out = repoman_getstatusoutput( - "egrep -q %s %s" % (headerstring, portage._shell_quote(_file)) - ) - if _out[0] == 0: - headers.append(_file) - - print("%s have headers that will change." % green(str(len(headers)))) - print( - "* Files with headers will" - " cause the manifests to be changed and committed separately." - ) - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - if updates or removed: - for x in sorted( - vcs_files_to_cps( - chain(updates, removed, manifests), - self.repo_settings.repodir, - scanner.repolevel, - scanner.reposplit, - scanner.categories, - ) - ): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) diff --git a/repoman/lib/repoman/modules/vcs/cvs/status.py b/repoman/lib/repoman/modules/vcs/cvs/status.py deleted file mode 100644 index 0d131c75a6c..00000000000 --- a/repoman/lib/repoman/modules/vcs/cvs/status.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -CVS module Status class submodule -""" - -import logging -import subprocess -import sys - -from repoman._portage import portage # pylint: disable=unused-import -from portage import os -from portage.const import BASH_BINARY -from portage.output import red, green -from portage import _unicode_encode, _unicode_decode - - -class Status: - """Performs status checks on the svn repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - try: - myf = open(checkdir + "/CVS/Entries", "r") - myl = myf.readlines() - myf.close() - except IOError: - self.qatracker.add_error("CVS/Entries.IO_error", checkdir + "/CVS/Entries") - return True - for l in myl: - if l[0] != "/": - continue - splitl = l[1:].split("/") - if not len(splitl): - continue - if splitl[0][-7:] == ".ebuild": - self.eadded.append(splitl[0][:-7]) - return True - - @staticmethod - def detect_conflicts(options): - """Determine if the checkout has cvs conflicts. - - TODO(antarus): Also this should probably not call sys.exit() as - repoman is run on >1 packages and one failure should not cause - subsequent packages to fail. - - Returns: - None (calls sys.exit on fatal problems) - """ - - cmd = ( - r"cvs -n up 2>/dev/null | " - r"egrep '^[^\?] .*' | " - r"egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'" - ) - msg = "Performing a %s with a little magic grep to check for updates." % green( - "cvs -n up" - ) - - logging.info(msg) - # Use Popen instead of getstatusoutput(), in order to avoid - # unicode handling problems (see bug #310789). - args = [BASH_BINARY, "-c", cmd] - args = [_unicode_encode(x) for x in args] - proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = _unicode_decode(proc.communicate()[0]) - proc.wait() - mylines = out.splitlines() - myupdates = [] - for line in mylines: - if not line: - continue - - # [ ] Unmodified (SVN) [U] Updates [P] Patches - # [M] Modified [A] Added [R] Removed / Replaced - # [D] Deleted - if line[0] not in " UPMARD": - # Stray Manifest is fine, we will readd it anyway. - if line[0] == "?" and line[1:].lstrip() == "Manifest": - continue - logging.error( - red( - "!!! Please fix the following issues reported " - "from cvs: %s" % green("(U,P,M,A,R,D are ok)") - ) - ) - logging.error(red("!!! Note: This is a pretend/no-modify pass...")) - logging.error(out) - sys.exit(1) - elif line[0] in "UP": - myupdates.append(line[2:]) - - if myupdates: - logging.info(green("Fetching trivial updates...")) - if options.pretend: - logging.info("(cvs update " + " ".join(myupdates) + ")") - retval = os.EX_OK - else: - retval = os.system("cvs update " + " ".join(myupdates)) - if retval != os.EX_OK: - logging.fatal("!!! cvs exited with an error. Terminating.") - sys.exit(retval) - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - return False - - @staticmethod - def isVcsDir(dirname): - """Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return dirname in ["CVS"] diff --git a/repoman/lib/repoman/modules/vcs/git/__init__.py b/repoman/lib/repoman/modules/vcs/git/__init__.py deleted file mode 100644 index c3005fc4186..00000000000 --- a/repoman/lib/repoman/modules/vcs/git/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Git (git) plug-in module for portage. -Performs variaous git actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "git", - "description": doc, - "provides": { - "git-module": { - "name": "git_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": False, - "needs_keyword_expansion": False, - }, - "git-changes": { - "name": "git_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/git/changes.py b/repoman/lib/repoman/modules/vcs/git/changes.py deleted file mode 100644 index 3430cf3ecd6..00000000000 --- a/repoman/lib/repoman/modules/vcs/git/changes.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Git module Changes class submodule -""" - -import logging -import sys - -from repoman._portage import portage # pylint: disable=unused-import -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen - -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.process import spawn -from portage.util import writemsg_level - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "git" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - - def _scan(self, _reindex=None): - """ - VCS type scan function, looks for all detectable changes - - @param _reindex: ensure that the git index reflects the state on - disk for files returned by git diff-index (this parameter is - used in recursive calls and it's not intended to be used for - any other reason) - @type _reindex: bool - """ - # Automatically reindex for commit mode, but not for other modes - # were the user might not want changes to be staged in the index. - if _reindex is None and self.options.mode == "commit": - _reindex = True - - with repoman_popen( - "git diff-index --name-only " "--relative --diff-filter=M HEAD" - ) as f: - changed = f.readlines() - self.changed = ["./" + elem[:-1] for elem in changed] - del changed - - with repoman_popen( - "git diff-index --name-only " "--relative --diff-filter=A HEAD" - ) as f: - new = f.readlines() - self.new = ["./" + elem[:-1] for elem in new] - del new - - with repoman_popen( - "git diff-index --name-only " "--relative --diff-filter=D HEAD" - ) as f: - removed = f.readlines() - self.removed = ["./" + elem[:-1] for elem in removed] - del removed - if _reindex and (self.changed or self.new or self.removed): - self.update_index([], self.changed + self.new + self.removed) - self._scan(_reindex=False) - - @property - def unadded(self): - """VCS method of getting the unadded files in the repository""" - if self._unadded is not None: - return self._unadded - # get list of files not under version control or missing - with repoman_popen("git ls-files --others") as f: - unadded = f.readlines() - self._unadded = ["./" + elem[:-1] for elem in unadded] - del unadded - return self._unadded - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) - - def update_index(self, mymanifests, myupdates): - """Update the vcs's modified index if it is needed - - @param mymanifests: manifest files updated - @param myupdates: other files updated""" - # It's not safe to use the git commit -a option since there might - # be some modified files elsewhere in the working tree that the - # user doesn't want to commit. Therefore, call git update-index - # in order to ensure that the index is updated with the latest - # versions of all new and modified files in the relevant portion - # of the working tree. - myfiles = mymanifests + myupdates - myfiles.sort() - update_index_cmd = ["git", "update-index", "--add", "--remove"] - update_index_cmd.extend(f.lstrip("./") for f in myfiles) - if self.options.pretend: - print("(%s)" % (" ".join(update_index_cmd),)) - else: - retval = spawn(update_index_cmd, env=os.environ) - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, - noiselevel=-1, - ) - sys.exit(retval) - - def commit(self, myfiles, commitmessagefile): - """Git commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - """ - retval = super(Changes, self).commit(myfiles, commitmessagefile) - if retval != os.EX_OK: - if ( - self.repo_settings.repo_config.sign_commit - and not self.vcs_settings.status.supports_gpg_sign() - ): - # Inform user that newer git is needed (bug #403323). - logging.error("Git >=1.7.9 is required for signed commits!") - return retval diff --git a/repoman/lib/repoman/modules/vcs/git/status.py b/repoman/lib/repoman/modules/vcs/git/status.py deleted file mode 100644 index ec2e3a65574..00000000000 --- a/repoman/lib/repoman/modules/vcs/git/status.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Git module Status class submodule -""" - -import re - -from repoman._portage import portage -from portage import os - -# pylint: disable=ungrouped-imports -from repoman._subprocess import repoman_popen, repoman_getstatusoutput - - -class Status: - """Performs status checks on the git repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the git status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - with repoman_popen( - "git ls-files --others %s" % (portage._shell_quote(checkdir_relative),) - ) as myf: - for l in myf: - if l[:-1][-7:] == ".ebuild": - self.qatracker.add_error( - "ebuild.notadded", os.path.join(xpkg, os.path.basename(l[:-1])) - ) - return True - - @staticmethod - def detect_conflicts(options): - """Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - """ - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - status, cmd_output = repoman_getstatusoutput("git --version") - cmd_output = cmd_output.split() - if cmd_output: - version = re.match(r"^(\d+)\.(\d+)\.(\d+)", cmd_output[-1]) - if version is not None: - version = [int(x) for x in version.groups()] - if ( - version[0] > 1 - or (version[0] == 1 and version[1] > 7) - or (version[0] == 1 and version[1] == 7 and version[2] >= 9) - ): - return True - return False - - @staticmethod - def isVcsDir(dirname): - """Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return dirname in [".git"] diff --git a/repoman/lib/repoman/modules/vcs/hg/__init__.py b/repoman/lib/repoman/modules/vcs/hg/__init__.py deleted file mode 100644 index 4b96b04c826..00000000000 --- a/repoman/lib/repoman/modules/vcs/hg/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Mercurial (hg) plug-in module for portage. -Performs variaous mercurial actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "hg", - "description": doc, - "provides": { - "hg-module": { - "name": "hg_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": False, - "needs_keyword_expansion": False, - }, - "hg-changes": { - "name": "hg_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/hg/changes.py b/repoman/lib/repoman/modules/vcs/hg/changes.py deleted file mode 100644 index 3ce1763b762..00000000000 --- a/repoman/lib/repoman/modules/vcs/hg/changes.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Mercurial module Changes class submodule -""" - -from repoman._portage import portage # pylint: disable=unused-import -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen - -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.process import spawn - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "hg" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - """VCS type scan function, looks for all detectable changes""" - with repoman_popen("hg status --no-status --modified .") as f: - changed = f.readlines() - self.changed = ["./" + elem.rstrip() for elem in changed] - del changed - - with repoman_popen("hg status --no-status --added .") as f: - new = f.readlines() - self.new = ["./" + elem.rstrip() for elem in new] - del new - - with repoman_popen("hg status --no-status --removed .") as f: - removed = f.readlines() - self.removed = ["./" + elem.rstrip() for elem in removed] - del removed - - @property - def unadded(self): - """VCS method of getting the unadded files in the repository""" - if self._unadded is not None: - return self._unadded - with repoman_popen("hg status --no-status --unknown .") as f: - unadded = f.readlines() - self._unadded = ["./" + elem.rstrip() for elem in unadded] - del unadded - return self._unadded - - @property - def deleted(self): - """VCS method of getting the deleted files in the repository""" - if self._deleted is not None: - return self._deleted - # Mercurial doesn't handle manually deleted files as removed from - # the repository, so the user need to remove them before commit, - # using "hg remove [FILES]" - with repoman_popen("hg status --no-status --deleted .") as f: - deleted = f.readlines() - self._deleted = ["./" + elem.rstrip() for elem in deleted] - del deleted - return self._deleted - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) - - def commit(self, myfiles, commitmessagefile): - """Hg commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - """ - commit_cmd = [] - commit_cmd.append(self.vcs) - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["--logfile", commitmessagefile]) - commit_cmd.extend(myfiles) - - if self.options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) - return 0 - else: - retval = spawn(commit_cmd, env=self.repo_settings.commit_env) - return retval diff --git a/repoman/lib/repoman/modules/vcs/hg/status.py b/repoman/lib/repoman/modules/vcs/hg/status.py deleted file mode 100644 index 2b56f2e1f21..00000000000 --- a/repoman/lib/repoman/modules/vcs/hg/status.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Mercurial module Status class submodule -""" - -from repoman._portage import portage -from portage import os - -# pylint: disable=ungrouped-imports -from repoman._subprocess import repoman_popen - - -class Status: - """Performs status checks on the svn repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - myf = repoman_popen( - "hg status --no-status --unknown %s" - % (portage._shell_quote(checkdir_relative),) - ) - for l in myf: - if l[:-1][-7:] == ".ebuild": - self.qatracker.add_error( - "ebuild.notadded", os.path.join(xpkg, os.path.basename(l[:-1])) - ) - myf.close() - return True - - @staticmethod - def detect_conflicts(options): - """Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - """ - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - return False - - @staticmethod - def isVcsDir(dirname): - """Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return dirname in [".hg"] diff --git a/repoman/lib/repoman/modules/vcs/settings.py b/repoman/lib/repoman/modules/vcs/settings.py deleted file mode 100644 index b186b70442c..00000000000 --- a/repoman/lib/repoman/modules/vcs/settings.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Repoman VCSSettings modules -""" - -import logging -import sys - -from portage.output import red - -from repoman.modules.vcs import module_controller, module_names -from repoman.modules.vcs.vcs import FindVCS -from repoman.qa_tracker import QATracker - - -class VCSSettings: - """Holds various VCS settings""" - - def __init__(self, options=None, repoman_settings=None, repo_settings=None): - """Class init function - - @param options: the run time cli options - @param repoman_settings: portage.config settings instance - @param repo_settings: RepoSettings instance - """ - self.options = options - self.repoman_settings = repoman_settings - self.repo_settings = repo_settings - if options.vcs: - if options.vcs in module_names: - self.vcs = options.vcs - else: - self.vcs = None - else: - vcses = FindVCS() - if len(vcses) > 1: - print( - red( - "*** Ambiguous workdir -- more than one VCS found" - " at the same depth: %s." % ", ".join(vcses) - ) - ) - print( - red( - "*** Please either clean up your workdir" - " or specify --vcs option." - ) - ) - sys.exit(1) - elif vcses: - self.vcs = vcses[0] - else: - self.vcs = None - - if options.if_modified == "y" and self.vcs is None: - logging.info( - "Not in a version controlled repository; " "disabling --if-modified." - ) - options.if_modified = "n" - - # initialize our instance placeholders - self._status = None - self._changes = None - # get our vcs plugin controller and available module names - self.module_controller = module_controller - self.module_names = module_names - - # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075). - if str(self.vcs) in self.module_controller.parents: - self.vcs_preserves_mtime = module_controller.modules[ - "%s_status" % self.vcs - ]["vcs_preserves_mtime"] - else: - self.vcs_preserves_mtime = False - logging.error("VCSSettings: Unknown VCS type: %s", self.vcs) - logging.error("Available modules: %s", module_controller.parents) - - self.needs_keyword_expansion = module_controller.modules[ - "%s_status" % self.vcs - ]["needs_keyword_expansion"] - self.vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split() - self.vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS") - if self.vcs_global_opts is None: - if self.vcs in ("cvs", "svn"): - self.vcs_global_opts = "-q" - else: - self.vcs_global_opts = "" - self.vcs_global_opts = self.vcs_global_opts.split() - - if options.mode == "commit" and not options.pretend and not self.vcs: - logging.info( - "Not in a version controlled repository; " "enabling pretend mode." - ) - options.pretend = True - self.qatracker = QATracker() - self.eadded = [] - - @property - def status(self): - """Initializes and returns the class instance - of the vcs's Status class""" - if not self._status: - status = self.module_controller.get_class("%s_status" % self.vcs) - self._status = status(self.qatracker, self.eadded) - return self._status - - @property - def changes(self): - """Initializes and returns the class instance - of the vcs's Changes class""" - if not self._changes: - changes = self.module_controller.get_class("%s_changes" % self.vcs) - self._changes = changes(self.options, self.repo_settings) - return self._changes diff --git a/repoman/lib/repoman/modules/vcs/svn/__init__.py b/repoman/lib/repoman/modules/vcs/svn/__init__.py deleted file mode 100644 index f8da39363c8..00000000000 --- a/repoman/lib/repoman/modules/vcs/svn/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Subversion (svn) plug-in module for portage. -Performs variaous subversion actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - "name": "svn", - "description": doc, - "provides": { - "svn-module": { - "name": "svn_status", - "sourcefile": "status", - "class": "Status", - "description": doc, - "functions": ["check", "supports_gpg_sign", "detect_conflicts"], - "func_desc": {}, - "vcs_preserves_mtime": False, - "needs_keyword_expansion": True, - }, - "svn-changes": { - "name": "svn_changes", - "sourcefile": "changes", - "class": "Changes", - "description": doc, - "functions": ["scan"], - "func_desc": {}, - }, - }, -} diff --git a/repoman/lib/repoman/modules/vcs/svn/changes.py b/repoman/lib/repoman/modules/vcs/svn/changes.py deleted file mode 100644 index e437325dbc7..00000000000 --- a/repoman/lib/repoman/modules/vcs/svn/changes.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -Subversion module Changes class submodule -""" - -from itertools import chain - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._subprocess import repoman_getstatusoutput -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman._portage import portage -from portage import os -from portage.output import green -from portage.package.ebuild.digestgen import digestgen - - -class Changes(ChangesBase): - """Class object to scan and hold the resultant data - for all changes to process. - """ - - vcs = "svn" - - def __init__(self, options, repo_settings): - """Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - """ - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - """VCS type scan function, looks for all detectable changes""" - with repoman_popen("svn status") as f: - svnstatus = f.readlines() - self.changed = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem and elem[:1] in "MR" - ] - self.new = [ - "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") - ] - self.removed = [ - "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D") - ] - - @property - def expansion(self): - """VCS method of getting the expanded keywords in the repository""" - if self._expansion is not None: - return self._expansion - # Subversion expands keywords specified in svn:keywords properties. - with repoman_popen("svn propget -R svn:keywords") as f: - props = f.readlines() - self._expansion = dict( - ("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) - for prop in props - if " - " in prop - ) - del props - return self._expansion - - @property - def unadded(self): - """VCS method of getting the unadded files in the repository""" - if self._unadded is not None: - return self._unadded - with repoman_popen("svn status --no-ignore") as f: - svnstatus = f.readlines() - self._unadded = [ - "./" + elem.rstrip().split()[1] - for elem in svnstatus - if elem.startswith("?") or elem.startswith("I") - ] - del svnstatus - return self._unadded - - def thick_manifest(self, updates, headers, no_expansion, expansion): - """Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - """ - svn_keywords = dict( - (k.lower(), k) - for k in [ - "Rev", - "Revision", - "LastChangedRevision", - "Date", - "LastChangedDate", - "Author", - "LastChangedBy", - "URL", - "HeadURL", - "Id", - "Header", - ] - ) - - for _file in updates: - # for SVN, expansion contains files that are included in expansion - if _file not in expansion: - continue - - # Subversion keywords are case-insensitive - # in svn:keywords properties, - # but case-sensitive in contents of files. - enabled_keywords = [] - for k in expansion[_file]: - keyword = svn_keywords.get(k.lower()) - if keyword is not None: - enabled_keywords.append(keyword) - - headerstring = r"'\$(%s).*\$'" % "|".join(enabled_keywords) - - _out = repoman_getstatusoutput( - "egrep -q %s %s" % (headerstring, portage._shell_quote(_file)) - ) - if _out[0] == 0: - headers.append(_file) - - print("%s have headers that will change." % green(str(len(headers)))) - print( - "* Files with headers will" - " cause the manifests to be changed and committed separately." - ) - - def digest_regen( - self, updates, removed, manifests, scanner, broken_changelog_manifests - ): - """Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - """ - if updates or removed: - for x in sorted( - vcs_files_to_cps( - chain(updates, removed, manifests), - scanner.repo_settings.repodir, - scanner.repolevel, - scanner.reposplit, - scanner.categories, - ) - ): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen( - mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb - ) diff --git a/repoman/lib/repoman/modules/vcs/svn/status.py b/repoman/lib/repoman/modules/vcs/svn/status.py deleted file mode 100644 index 2e6ee927d24..00000000000 --- a/repoman/lib/repoman/modules/vcs/svn/status.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -Subversion module Status class submodule -""" - -import logging -import subprocess -import sys - -from repoman._portage import portage -from portage import os -from portage.const import BASH_BINARY -from portage.output import red, green -from portage import _unicode_encode, _unicode_decode - -# pylint: disable=ungrouped-imports -from repoman._subprocess import repoman_popen - - -class Status: - """Performs status checks on the svn repository""" - - def __init__(self, qatracker, eadded): - """Class init - - @param qatracker: QATracker class instance - @param eadded: list - """ - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - """Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - """ - try: - myf = repoman_popen( - "svn status --depth=files --verbose " + portage._shell_quote(checkdir) - ) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[:1] == "?": - continue - if l[:7] == " >": - # tree conflict, new in subversion 1.6 - continue - l = l.split()[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - try: - myf = repoman_popen("svn status " + portage._shell_quote(checkdir)) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[0] == "A": - l = l.rstrip().split(" ")[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - return True - - @staticmethod - def detect_conflicts(options): - """Determine if the checkout has problems like cvs conflicts. - - If you want more vcs support here just keep adding if blocks... - This could be better. - - TODO(antarus): Also this should probably not call sys.exit() as - repoman is run on >1 packages and one failure should not cause - subsequent packages to fail. - - Args: - vcs - A string identifying the version control system in use - Returns: boolean - (calls sys.exit on fatal problems) - """ - - cmd = "svn status -u 2>&1 | egrep -v '^. +.*/digest-[^/]+' | head -n-1" - msg = "Performing a %s with a little magic grep to check for updates." % green( - "svn status -u" - ) - - logging.info(msg) - # Use Popen instead of getstatusoutput(), in order to avoid - # unicode handling problems (see bug #310789). - args = [BASH_BINARY, "-c", cmd] - args = [_unicode_encode(x) for x in args] - proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = _unicode_decode(proc.communicate()[0]) - proc.wait() - mylines = out.splitlines() - myupdates = [] - for line in mylines: - if not line: - continue - - # [ ] Unmodified (SVN) [U] Updates [P] Patches - # [M] Modified [A] Added [R] Removed / Replaced - # [D] Deleted - if line[0] not in " UPMARD": - # Stray Manifest is fine, we will readd it anyway. - if line[0] == "?" and line[1:].lstrip() == "Manifest": - continue - logging.error( - red( - "!!! Please fix the following issues reported " - "from cvs: %s" % green("(U,P,M,A,R,D are ok)") - ) - ) - logging.error(red("!!! Note: This is a pretend/no-modify pass...")) - logging.error(out) - sys.exit(1) - elif line[8] == "*": - myupdates.append(line[9:].lstrip(" 1234567890")) - - if myupdates: - logging.info(green("Fetching trivial updates...")) - if options.pretend: - logging.info("(svn update " + " ".join(myupdates) + ")") - retval = os.EX_OK - else: - retval = os.system("svn update " + " ".join(myupdates)) - if retval != os.EX_OK: - logging.fatal("!!! svn exited with an error. Terminating.") - sys.exit(retval) - return False - - @staticmethod - def supports_gpg_sign(): - """Does this vcs system support gpg commit signatures - - @returns: Boolean - """ - return False - - @staticmethod - def isVcsDir(dirname): - """Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - """ - return dirname in [".svn"] diff --git a/repoman/lib/repoman/modules/vcs/vcs.py b/repoman/lib/repoman/modules/vcs/vcs.py deleted file mode 100644 index 1ee8b52ffb8..00000000000 --- a/repoman/lib/repoman/modules/vcs/vcs.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding:utf-8 -*- - -import collections -import logging - -from itertools import chain - -from portage import os - - -_vcs_type = collections.namedtuple("_vcs_type", "name dir_name file_name") - -_FindVCS_data = ( - _vcs_type(name="git", dir_name=".git", file_name=".git"), - _vcs_type(name="bzr", dir_name=".bzr", file_name=""), - _vcs_type(name="hg", dir_name=".hg", file_name=""), - _vcs_type(name="svn", dir_name=".svn", file_name=""), -) - - -def FindVCS(cwd=None): - """ - Try to figure out in what VCS' working tree we are. - - @param cwd: working directory (default is os.getcwd()) - @type cwd: str - @return: list of strings describing the discovered vcs types - @rtype: list - """ - - if cwd is None: - cwd = os.getcwd() - - outvcs = [] - - def seek(depth=None): - """Seek for VCSes that have a top-level data directory only. - - @param depth: integer - @returns: list of strings - """ - retvcs = [] - pathprep = cwd - - while depth is None or depth > 0: - for vcs_type in _FindVCS_data: - vcs_dir = os.path.join(pathprep, vcs_type.dir_name) - if os.path.isdir(vcs_dir): - logging.debug( - "FindVCS: found %(name)s dir: %(vcs_dir)s" - % {"name": vcs_type.name, "vcs_dir": os.path.abspath(vcs_dir)} - ) - retvcs.append(vcs_type.name) - elif vcs_type.file_name: - vcs_file = os.path.join(pathprep, vcs_type.file_name) - if os.path.exists(vcs_file): - logging.debug( - "FindVCS: found %(name)s file: %(vcs_file)s" - % { - "name": vcs_type.name, - "vcs_file": os.path.abspath(vcs_file), - } - ) - retvcs.append(vcs_type.name) - - if retvcs: - break - pathprep = os.path.join(pathprep, "..") - if os.path.realpath(pathprep).strip("/") == "": - break - if depth is not None: - depth = depth - 1 - - return retvcs - - # Level zero VCS-es. - if os.path.isdir(os.path.join(cwd, "CVS")): - outvcs.append("cvs") - if os.path.isdir(".svn"): # <1.7 - outvcs.append(os.path.join(cwd, "svn")) - - # If we already found one of 'level zeros', just take a quick look - # at the current directory. Otherwise, seek parents till we get - # something or reach root. - if outvcs: - outvcs.extend(seek(1)) - else: - outvcs = seek() - - if len(outvcs) > 1: - # eliminate duplicates, like for svn in bug #391199 - outvcs = list(set(outvcs)) - - return outvcs - - -def vcs_files_to_cps(vcs_file_iter, repodir, repolevel, reposplit, categories): - """ - Iterate over the given modified file paths returned from the vcs, - and return a frozenset containing category/pn strings for each - modified package. - """ - - modified_cps = [] - - if repolevel == 3: - if reposplit[-2] in categories and next(vcs_file_iter, None) is not None: - modified_cps.append("/".join(reposplit[-2:])) - - elif repolevel == 2: - category = reposplit[-1] - if category in categories: - for filename in vcs_file_iter: - f_split = filename.split(os.sep) - # ['.', pn, ...] - if len(f_split) > 2: - modified_cps.append(category + "/" + f_split[1]) - - else: - # repolevel == 1 - for filename in vcs_file_iter: - f_split = filename.split(os.sep) - # ['.', category, pn, ...] - if len(f_split) > 3 and f_split[1] in categories: - modified_cps.append("/".join(f_split[1:3])) - - # Exclude packages that have been removed, since calling - # code assumes that the packages exist. - return frozenset( - x for x in frozenset(modified_cps) if os.path.exists(os.path.join(repodir, x)) - ) - - -def vcs_new_changed(relative_path, mychanged, mynew): - """Check if any vcs tracked file have been modified - - @param relative_path: - @param mychanged: iterable of changed files - @param mynew: iterable of new files - @returns boolean - """ - for x in chain(mychanged, mynew): - if x == relative_path: - return True - return False diff --git a/repoman/lib/repoman/profile.py b/repoman/lib/repoman/profile.py deleted file mode 100644 index b317189effc..00000000000 --- a/repoman/lib/repoman/profile.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding:utf-8 -*- - -from portage import normalize_path -from portage import os -from portage.output import red - - -class ProfileDesc: - __slots__ = ( - "abs_path", - "arch", - "status", - "sub_path", - "tree_path", - ) - - def __init__(self, arch, status, sub_path, tree_path): - self.arch = arch - self.status = status - if sub_path: - sub_path = normalize_path(sub_path.lstrip(os.sep)) - self.sub_path = sub_path - self.tree_path = tree_path - if tree_path: - self.abs_path = os.path.join(tree_path, "profiles", self.sub_path) - else: - self.abs_path = tree_path - - def __str__(self): - if self.sub_path: - return self.sub_path - return "empty profile" - - -valid_profile_types = frozenset(["dev", "exp", "stable"]) - - -def dev_profile_keywords(profiles): - """ - Create a set of KEYWORDS values that exist in 'dev' - profiles. These are used - to trigger a message notifying the user when they might - want to add the --include-dev option. - """ - type_arch_map = {} - for arch, arch_profiles in profiles.items(): - for prof in arch_profiles: - arch_set = type_arch_map.get(prof.status) - if arch_set is None: - arch_set = set() - type_arch_map[prof.status] = arch_set - arch_set.add(arch) - - dev_keywords = type_arch_map.get("dev", set()) - dev_keywords.update(["~" + arch for arch in dev_keywords]) - return frozenset(dev_keywords) - - -def setup_profile(profile_list): - # Ensure that profile sub_path attributes are unique. Process in reverse order - # so that profiles with duplicate sub_path from overlays will override - # profiles with the same sub_path from parent repos. - profiles = {} - profile_list.reverse() - profile_sub_paths = set() - for prof in profile_list: - if prof.sub_path in profile_sub_paths: - continue - profile_sub_paths.add(prof.sub_path) - profiles.setdefault(prof.arch, []).append(prof) - - # Use an empty profile for checking dependencies of - # packages that have empty KEYWORDS. - prof = ProfileDesc("**", "stable", "", "") - profiles.setdefault(prof.arch, []).append(prof) - return profiles - - -def check_profiles(profiles, archlist): - for x in archlist: - if x[0] == "~": - continue - if x not in profiles: - print( - red('"%s" doesn\'t have a valid profile listed in profiles.desc.' % x) - ) - print( - red( - 'You need to either "cvs update" your profiles dir' - " or follow this" - ) - ) - print(red("up with the " + x + " team.")) - print() diff --git a/repoman/lib/repoman/qa_data.py b/repoman/lib/repoman/qa_data.py deleted file mode 100644 index 0892c1f5ced..00000000000 --- a/repoman/lib/repoman/qa_data.py +++ /dev/null @@ -1,210 +0,0 @@ -# -*- coding:utf-8 -*- - -import logging -import os - -from _emerge.Package import Package - -# import our initialized portage instance -from repoman import _not_installed -from repoman._portage import portage -from repoman.config import load_config - - -class QAData: - def __init__(self): - # Create the main exported data variables - self.max_desc_len = None - self.allowed_filename_chars = None - self.qahelp = None - self.qacats = None - self.qawarnings = None - self.missingvars = None - self.allvars = None - self.valid_restrict = None - self.suspect_rdepend = None - self.suspect_virtual = None - self.ruby_deprecated = None - self.no_exec = None - - def load_repo_config(self, repopaths, options, valid_versions): - """Load the repository repoman qa_data.yml config - - @param repopaths: list of strings, The path of the repository being scanned - This could be a parent repository using the - repoman_masters layout.conf variable - """ - # add our base qahelp - repository_modules = options.experimental_repository_modules == "y" - if _not_installed: - cnfdir = os.path.realpath( - os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "cnf/qa_data", - ) - ) - else: - cnfdir = os.path.join( - portage.const.EPREFIX or "/", "usr/share/repoman/qa_data" - ) - repomanpaths = [os.path.join(cnfdir, _file_) for _file_ in os.listdir(cnfdir)] - logging.debug("QAData: cnfdir: %s, repomanpaths: %s", cnfdir, repomanpaths) - if repository_modules: - repopaths = [os.path.join(path, "qa_data.yaml") for path in repopaths] - elif _not_installed: - repopaths = [ - os.path.realpath( - os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "cnf/repository/qa_data.yaml", - ) - ) - ] - else: - repopaths = [ - os.path.join( - portage.const.EPREFIX or "/", - "usr/share/repoman/repository/qa_data.yaml", - ) - ] - infopaths = repomanpaths + repopaths - - qadata = load_config(infopaths, None, valid_versions) - if qadata == {}: - logging.error( - "QAData: Failed to load a valid 'qa_data.yaml' file at paths: %s", - infopaths, - ) - return False - self.max_desc_len = qadata.get("max_description_length", 80) - self.allowed_filename_chars = qadata.get( - "allowed_filename_chars", "a-zA-Z0-9._-+:" - ) - - self.qahelp = qadata["qahelp"] - logging.debug("qa_help primary keys: %s", sorted(self.qahelp)) - - self.qacats = [] - for x in sorted(self.qahelp): - for y in sorted(self.qahelp[x]): - self.qacats.append(".".join([x, y])) - self.qacats.sort() - - self.qawarnings = set(qadata.get("qawarnings", [])) - if options.experimental_inherit == "y": - # This is experimental, so it's non-fatal. - self.qawarnings.add("inherit.missing") - - self.missingvars = qadata.get("missingvars", []) - logging.debug("QAData: missingvars: %s", self.missingvars) - self.allvars = set(portage.auxdbkeys) - self.allvars.update(Package.metadata_keys) - self.allvars = sorted(self.allvars) - - for x in self.missingvars: - x += ".missing" - if x not in self.qacats: - logging.warning( - 'QAData: * missingvars values need to be added to qahelp ("%s")' % x - ) - self.qacats.append(x) - self.qawarnings.add(x) - - self.valid_restrict = frozenset(qadata.get("valid_restrict", [])) - - self.suspect_rdepend = frozenset(qadata.get("suspect_rdepend", [])) - - self.suspect_virtual = qadata.get("suspect_virtual", {}) - - self.ruby_deprecated = frozenset(qadata.get("ruby_deprecated", [])) - - # file.executable - self.no_exec = frozenset(qadata.get("no_exec_files", [])) - logging.debug("QAData: completed loading file: %s", repopaths) - return True - - -def format_qa_output(formatter, fails, dofull, dofail, options, qawarnings): - """Helper function that formats output properly - - @param formatter: an instance of Formatter - @type formatter: Formatter - @param fails: dict of qa status failures - @type fails: dict - @param dofull: Whether to print full results or a summary - @type dofull: boolean - @param dofail: Whether failure was hard or soft - @type dofail: boolean - @param options: The command-line options provided to repoman - @type options: Namespace - @param qawarnings: the set of warning types - @type qawarnings: set - @return: None (modifies formatter) - """ - full = options.mode == "full" - # we only want key value pairs where value > 0 - for category in sorted(fails): - number = len(fails[category]) - formatter.add_literal_data(" " + category) - spacing_width = 30 - len(category) - if category in qawarnings: - formatter.push_style("WARN") - else: - formatter.push_style("BAD") - formatter.add_literal_data(" [fatal]") - spacing_width -= 8 - - formatter.add_literal_data(" " * spacing_width) - formatter.add_literal_data("%s" % number) - formatter.pop_style() - formatter.add_line_break() - if not dofull: - if not full and dofail and category in qawarnings: - # warnings are considered noise when there are failures - continue - fails_list = fails[category] - if not full and len(fails_list) > 12: - fails_list = fails_list[:12] - for failure in fails_list: - formatter.add_literal_data(" " + failure) - formatter.add_line_break() - - -def format_qa_output_column(formatter, fails, dofull, dofail, options, qawarnings): - """Helper function that formats output in a machine-parseable column format - - @param formatter: an instance of Formatter - @type formatter: Formatter - @param fails: dict of qa status failures - @type fails: dict - @param dofull: Whether to print full results or a summary - @type dofull: boolean - @param dofail: Whether failure was hard or soft - @type dofail: boolean - @param options: The command-line options provided to repoman - @type options: Namespace - @param qawarnings: the set of warning types - @type qawarnings: set - @return: None (modifies formatter) - """ - full = options.mode == "full" - for category in sorted(fails): - number = len(fails[category]) - formatter.add_literal_data("NumberOf " + category + " ") - if category in qawarnings: - formatter.push_style("WARN") - else: - formatter.push_style("BAD") - formatter.add_literal_data("%s" % number) - formatter.pop_style() - formatter.add_line_break() - if not dofull: - if not full and dofail and category in qawarnings: - # warnings are considered noise when there are failures - continue - fails_list = fails[category] - if not full and len(fails_list) > 12: - fails_list = fails_list[:12] - for failure in fails_list: - formatter.add_literal_data(category + " " + failure) - formatter.add_line_break() diff --git a/repoman/lib/repoman/qa_tracker.py b/repoman/lib/repoman/qa_tracker.py deleted file mode 100644 index 6cb5f320704..00000000000 --- a/repoman/lib/repoman/qa_tracker.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import sys - - -class QATracker: - """Track all occurrances of Q/A problems detected""" - - def __init__(self, qacats=None, qawarnings=None): - self.fails = {} - self.warns = {} - self.qacats = qacats - self.qawarnings = qawarnings - - def add_error(self, detected_qa, info): - """Add the Q/A error to the database of detected problems - - @param detected_qa: string, member of qa_data.qacats list - @param info: string, details of the detected problem - """ - if detected_qa not in self.qacats: - logging.error( - "QATracker: Exiting on error. Unknown detected_qa type passed " - "in to add_error(): %s, %s" % (detected_qa, info) - ) - sys.exit(1) - try: - self.fails[detected_qa].append(info) - except KeyError: - self.fails[detected_qa] = [info] - - def add_warning(self, detected_qa, info): - """Add the Q/A warning to the database of detected problems - - @param detected_qa: string, member of qa_data.qawarnings list - @param info: string, details of the detected problem - """ - if detected_qa not in self.qawarnings: - logging.error( - "QATracker: Exiting on error. Unknown detected_qa type passed " - "in to add_warning(): %s, %s" % (detected_qa, info) - ) - sys.exit(1) - try: - self.warns[detected_qa].append(info) - except KeyError: - self.warns[detected_qa] = [info] diff --git a/repoman/lib/repoman/repos.py b/repoman/lib/repoman/repos.py deleted file mode 100644 index d89e0508a26..00000000000 --- a/repoman/lib/repoman/repos.py +++ /dev/null @@ -1,377 +0,0 @@ -# -*- coding:utf-8 -*- - - -import io -import logging -import re -import sys -import textwrap - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.checksum import get_valid_checksum_keys -from portage.repository.config import allow_profile_repo_deps - -# pylint: disable=ungrouped-imports -from repoman.errors import err -from repoman.profile import ProfileDesc, valid_profile_types - -GPG_KEY_ID_REGEX = r"(0x)?([0-9a-fA-F]{8}){1,5}!?" -bad = portage.output.create_color_func("BAD") - - -class RepoSettings: - """Holds our repo specific settings""" - - def __init__( - self, - config_root, - portdir, - portdir_overlay, - repoman_settings=None, - vcs_settings=None, - options=None, - qadata=None, - ): - self.config_root = config_root - self.repoman_settings = repoman_settings - self.vcs_settings = vcs_settings - - self.repositories = self.repoman_settings.repositories - - # Ensure that current repository is in the list of enabled repositories. - self.repodir = os.path.realpath(portdir_overlay) - try: - self.repositories.get_repo_for_location(self.repodir) - except KeyError: - self._add_repo(config_root, portdir_overlay) - - # Determine the master config loading list - self.masters_list = [] - # get out repo masters value - masters = self.repositories.get_repo_for_location(self.repodir).masters - for repo in masters: - self.masters_list.append(os.path.join(repo.location, "metadata", "repoman")) - self.masters_list.append(os.path.join(self.repodir, "metadata", "repoman")) - - logging.debug("RepoSettings: init(); load qadata") - # load the repo specific configuration - self.qadata = qadata - if not self.qadata.load_repo_config( - self.masters_list, options, repoman_settings.valid_versions - ): - logging.error("Aborting...") - sys.exit(1) - logging.debug("RepoSettings: qadata loaded: %s", qadata.no_exec) - - self.root = self.repoman_settings["EROOT"] - self.trees = { - self.root: {"porttree": portage.portagetree(settings=self.repoman_settings)} - } - self.portdb = self.trees[self.root]["porttree"].dbapi - - # Constrain dependency resolution to the master(s) - # that are specified in layout.conf. - self.repo_config = self.repositories.get_repo_for_location(self.repodir) - self.portdb.porttrees = list(self.repo_config.eclass_db.porttrees) - self.portdir = self.portdb.porttrees[0] - self.commit_env = os.environ.copy() - # list() is for iteration on a copy. - for repo in list(self.repositories): - # all paths are canonical - if repo.location not in self.repo_config.eclass_db.porttrees: - del self.repositories[repo.name] - - if self.repo_config.sign_commit and options.mode in ( - "commit", - "fix", - "manifest", - ): - if vcs_settings.vcs: - func = getattr(self, "_vcs_gpg_%s" % vcs_settings.vcs) - func() - else: - logging.warning("No VCS type detected, unable to sign the commit") - - # In order to disable manifest signatures, repos may set - # "sign-manifests = false" in metadata/layout.conf. This - # can be used to prevent merge conflicts like those that - # thin-manifests is designed to prevent. - self.sign_manifests = ( - "sign" in self.repoman_settings.features and self.repo_config.sign_manifest - ) - - if ( - self.repo_config.sign_manifest - and self.repo_config.name == "gentoo" - and options.mode in ("commit",) - and not self.sign_manifests - ): - msg = ( - "The '%s' repository has manifest signatures enabled, " - "but FEATURES=sign is currently disabled. In order to avoid this " - "warning, enable FEATURES=sign in make.conf. Alternatively, " - "repositories can disable manifest signatures by setting " - "'sign-manifests = false' in metadata/layout.conf." - ) % (self.repo_config.name,) - for line in textwrap.wrap(msg, 60): - logging.warn(line) - - is_commit = options.mode in ("commit",) - valid_gpg_key = self.repoman_settings.get("PORTAGE_GPG_KEY") and re.match( - r"^%s$" % GPG_KEY_ID_REGEX, self.repoman_settings["PORTAGE_GPG_KEY"] - ) - - if self.sign_manifests and is_commit and not valid_gpg_key: - logging.error( - "PORTAGE_GPG_KEY value is invalid: %s" - % self.repoman_settings["PORTAGE_GPG_KEY"] - ) - sys.exit(1) - - manifest_hashes = self.repo_config.manifest_hashes - manifest_required_hashes = self.repo_config.manifest_required_hashes - if manifest_hashes is None: - manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS - manifest_required_hashes = manifest_hashes - - if options.mode in ("commit", "fix", "manifest"): - missing_required_hashes = manifest_required_hashes.difference( - manifest_hashes - ) - if missing_required_hashes: - msg = ( - "The 'manifest-hashes' setting in the '%s' repository's " - "metadata/layout.conf does not contain the '%s' hashes which " - "are listed in 'manifest-required-hashes'. Please fix that " - "file if you want to generate valid manifests for " - "this repository." - ) % (self.repo_config.name, " ".join(missing_required_hashes)) - for line in textwrap.wrap(msg, 70): - logging.error(line) - sys.exit(1) - - unsupported_hashes = manifest_hashes.difference(get_valid_checksum_keys()) - if unsupported_hashes: - msg = ( - "The 'manifest-hashes' setting in the '%s' repository's " - "metadata/layout.conf contains one or more hash types '%s' " - "which are not supported by this portage version. You will " - "have to upgrade portage if you want to generate valid " - "manifests for this repository." - ) % (self.repo_config.name, " ".join(sorted(unsupported_hashes))) - for line in textwrap.wrap(msg, 70): - logging.error(line) - sys.exit(1) - - def _add_repo(self, config_root, portdir_overlay): - self.repo_conf = portage.repository.config - self.repo_name = self.repo_conf.RepoConfig._read_valid_repo_name( - portdir_overlay - )[0] - self.layout_conf_data = self.repo_conf.parse_layout_conf(portdir_overlay)[0] - if self.layout_conf_data["repo-name"]: - self.repo_name = self.layout_conf_data["repo-name"] - tmp_conf_file = io.StringIO( - textwrap.dedent( - """ - [%s] - location = %s - """ - ) - % (self.repo_name, portdir_overlay) - ) - # Ensure that the repository corresponding to $PWD overrides a - # repository of the same name referenced by the existing PORTDIR - # or PORTDIR_OVERLAY settings. - self.repoman_settings["PORTDIR_OVERLAY"] = "%s %s" % ( - self.repoman_settings.get("PORTDIR_OVERLAY", ""), - portage._shell_quote(portdir_overlay), - ) - self.repositories = self.repo_conf.load_repository_config( - self.repoman_settings, extra_files=[tmp_conf_file] - ) - # We have to call the config constructor again so that attributes - # dependent on config.repositories are initialized correctly. - self.repoman_settings = portage.config( - config_root=config_root, local_config=False, repositories=self.repositories - ) - - ########## - # future vcs plugin functions - ########## - - def _vcs_gpg_bzr(self): - pass - - def _vcs_gpg_cvs(self): - pass - - def _vcs_gpg_git(self): - # NOTE: It's possible to use --gpg-sign=key_id to specify the key in - # the commit arguments. If key_id is unspecified, then it must be - # configured by `git config user.signingkey key_id`. - self.vcs_settings.vcs_local_opts.append("--gpg-sign") - if self.repoman_settings.get("PORTAGE_GPG_DIR"): - # Pass GNUPGHOME to git for bug #462362. - self.commit_env["GNUPGHOME"] = self.repoman_settings["PORTAGE_GPG_DIR"] - - # Pass GPG_TTY to git for bug #477728. - try: - self.commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno()) - except OSError: - pass - - def _vcs_gpg_hg(self): - pass - - def _vcs_gpg_svn(self): - pass - - -def list_checks(kwlist, liclist, uselist, repoman_settings): - liclist_deprecated = set() - if "DEPRECATED" in repoman_settings._license_manager._license_groups: - liclist_deprecated.update( - repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"]) - ) - - if not liclist: - logging.fatal("Couldn't find licenses?") - sys.exit(1) - - if not kwlist: - logging.fatal("Couldn't read KEYWORDS from arch.list") - sys.exit(1) - - if not uselist: - logging.fatal("Couldn't find use.desc?") - sys.exit(1) - return liclist_deprecated - - -def repo_metadata(portdb, repoman_settings): - # get lists of valid keywords, licenses, and use - kwlist = set() - liclist = set() - uselist = set() - profile_list = [] - global_pmasklines = [] - - for repo in portdb.repositories: - path = repo.location - - try: - liclist.update(os.listdir(os.path.join(path, "licenses"))) - except OSError: - pass - kwlist.update(portage.grabfile(os.path.join(path, "profiles", "arch.list"))) - - use_desc = portage.grabfile(os.path.join(path, "profiles", "use.desc")) - for x in use_desc: - x = x.split() - if x: - uselist.add(x[0]) - - expand_desc_dir = os.path.join(path, "profiles", "desc") - try: - expand_list = os.listdir(expand_desc_dir) - except OSError: - pass - else: - for fn in expand_list: - if not fn[-5:] == ".desc": - continue - use_prefix = fn[:-5].lower() + "_" - for x in portage.grabfile(os.path.join(expand_desc_dir, fn)): - x = x.split() - if x: - uselist.add(use_prefix + x[0]) - - global_pmasklines.append( - portage.util.grabfile_package( - os.path.join(path, "profiles", "package.mask"), - recursive=1, - verify_eapi=True, - allow_repo=allow_profile_repo_deps(repo), - ) - ) - - desc_path = os.path.join(path, "profiles", "profiles.desc") - try: - desc_file = io.open( - _unicode_encode(desc_path, encoding=_encodings["fs"], errors="strict"), - mode="r", - encoding=_encodings["repo.content"], - errors="replace", - ) - except EnvironmentError: - pass - else: - for i, x in enumerate(desc_file): - if x[0] == "#": - continue - arch = x.split() - if len(arch) == 0: - continue - if len(arch) != 3: - err( - 'wrong format: "%s" in %s line %d' - % ( - bad(x.strip()), - desc_path, - i + 1, - ) - ) - elif arch[0] not in kwlist: - err( - 'invalid arch: "%s" in %s line %d' - % ( - bad(arch[0]), - desc_path, - i + 1, - ) - ) - elif arch[2] not in valid_profile_types: - err( - 'invalid profile type: "%s" in %s line %d' - % ( - bad(arch[2]), - desc_path, - i + 1, - ) - ) - profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path) - if not os.path.isdir(profile_desc.abs_path): - logging.error( - "Invalid %s profile (%s) for arch %s in %s line %d", - arch[2], - arch[1], - arch[0], - desc_path, - i + 1, - ) - continue - if os.path.exists(os.path.join(profile_desc.abs_path, "deprecated")): - continue - profile_list.append(profile_desc) - desc_file.close() - - global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1) - global_pmaskdict = {} - for x in global_pmasklines: - global_pmaskdict.setdefault(x.cp, []).append(x) - del global_pmasklines - - return ( - kwlist, - liclist, - uselist, - profile_list, - global_pmaskdict, - list_checks(kwlist, liclist, uselist, repoman_settings), - ) diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py deleted file mode 100644 index 890950c65bc..00000000000 --- a/repoman/lib/repoman/scanner.py +++ /dev/null @@ -1,484 +0,0 @@ -# -*- coding:utf-8 -*- - -import logging -import portage - -from itertools import chain - -from portage import normalize_path -from portage import os -from portage._sets.base import InternalPackageSet -from portage.output import green -from portage.util.futures.extendedfutures import ExtendedFuture - -from repoman.metadata import get_metadata_xsd -from repoman.modules.commit import repochecks -from repoman.modules.commit import manifest -from repoman.profile import check_profiles, dev_profile_keywords, setup_profile -from repoman.repos import repo_metadata -from repoman.modules.scan.module import ModuleConfig -from repoman.modules.scan.scan import scan -from repoman.modules.vcs.vcs import vcs_files_to_cps - - -DATA_TYPES = {"dict": dict, "Future": ExtendedFuture, "list": list, "set": set} - - -class Scanner: - """Primary scan class. Operates all the small Q/A tests and checks""" - - def __init__( - self, repo_settings, myreporoot, config_root, options, vcs_settings, mydir, env - ): - """Class __init__""" - self.repo_settings = repo_settings - self.config_root = config_root - self.options = options - self.vcs_settings = vcs_settings - self.env = env - - # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to - # behave incrementally. - self.repoman_incrementals = tuple( - x for x in portage.const.INCREMENTALS if x != "ACCEPT_KEYWORDS" - ) - - self.categories = [] - for path in self.repo_settings.repo_config.eclass_db.porttrees: - self.categories.extend( - portage.util.grabfile(os.path.join(path, "profiles", "categories")) - ) - self.repo_settings.repoman_settings.categories = frozenset( - portage.util.stack_lists([self.categories], incremental=1) - ) - self.categories = self.repo_settings.repoman_settings.categories - - self.portdb = repo_settings.portdb - self.portdb.settings = self.repo_settings.repoman_settings - - digest_only = ( - self.options.mode != "manifest-check" and self.options.digest == "y" - ) - self.generate_manifest = digest_only or self.options.mode in ( - "manifest", - "commit", - "fix", - ) - - # We really only need to cache the metadata that's necessary for visibility - # filtering. Anything else can be discarded to reduce memory consumption. - if not self.generate_manifest: - # Don't do this when generating manifests, since that uses - # additional keys if spawn_nofetch is called (RESTRICT and - # DEFINED_PHASES). - self.portdb._aux_cache_keys.clear() - self.portdb._aux_cache_keys.update( - ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"] - ) - - self.reposplit = myreporoot.split(os.path.sep) - self.repolevel = len(self.reposplit) - - if self.options.mode == "commit": - repochecks.commit_check(self.repolevel, self.reposplit) - repochecks.conflict_check(self.vcs_settings, self.options) - - # Make startdir relative to the canonical repodir, so that we can pass - # it to digestgen and it won't have to be canonicalized again. - if self.repolevel == 1: - startdir = self.repo_settings.repodir - else: - startdir = normalize_path(mydir) - startdir = os.path.join( - self.repo_settings.repodir, - *startdir.split(os.sep)[-2 - self.repolevel + 3 :] - ) - - # get lists of valid keywords, licenses, and use - new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings) - ( - kwlist, - liclist, - uselist, - profile_list, - global_pmaskdict, - liclist_deprecated, - ) = new_data - self.repo_metadata = { - "kwlist": kwlist, - "liclist": liclist, - "uselist": uselist, - "profile_list": profile_list, - "pmaskdict": global_pmaskdict, - "lic_deprecated": liclist_deprecated, - "package.deprecated": InternalPackageSet( - initial_atoms=portage.util.stack_lists( - [ - portage.util.grabfile_package( - os.path.join(path, "profiles", "package.deprecated"), - recursive=True, - ) - for path in self.portdb.porttrees - ], - incremental=True, - ) - ), - } - - self.repo_settings.repoman_settings["PORTAGE_ARCHLIST"] = " ".join( - sorted(kwlist) - ) - self.repo_settings.repoman_settings.backup_changes("PORTAGE_ARCHLIST") - - profiles = setup_profile(profile_list) - - check_profiles(profiles, self.repo_settings.repoman_settings.archlist()) - - scanlist = scan( - self.repolevel, - self.reposplit, - startdir, - self.categories, - self.repo_settings, - ) - - self.dev_keywords = dev_profile_keywords(profiles) - - self.qatracker = self.vcs_settings.qatracker - - if ( - self.options.echangelog is None - and self.repo_settings.repo_config.update_changelog - ): - self.options.echangelog = "y" - - if self.vcs_settings.vcs is None: - self.options.echangelog = "n" - - # Initialize the ModuleConfig class here - # TODO Add layout.conf masters repository.yml config to the list to load/stack - self.moduleconfig = ModuleConfig( - self.repo_settings.masters_list, - self.repo_settings.repoman_settings.valid_versions, - repository_modules=self.options.experimental_repository_modules == "y", - ) - - checks = {} - # The --echangelog option causes automatic ChangeLog generation, - # which invalidates changelog.ebuildadded and changelog.missing - # checks. - # Note: Some don't use ChangeLogs in distributed SCMs. - # It will be generated on server side from scm log, - # before package moves to the rsync server. - # This is needed because they try to avoid merge collisions. - # Gentoo's Council decided to always use the ChangeLog file. - # TODO: shouldn't this just be switched on the repo, iso the VCS? - is_echangelog_enabled = self.options.echangelog in ("y", "force") - self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ("cvs", "svn") - checks["changelog"] = ( - not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn - ) - - if self.options.mode == "manifest" or self.options.quiet: - pass - elif self.options.pretend: - print(green("\nRepoMan does a once-over of the neighborhood...")) - else: - print(green("\nRepoMan scours the neighborhood...")) - - self.changed = self.vcs_settings.changes - # bypass unneeded VCS operations if not needed - if self.options.if_modified == "y" or self.options.mode not in ( - "manifest", - "manifest-check", - ): - self.changed.scan() - - self.have = { - "pmasked": False, - "dev_keywords": False, - } - - # NOTE: match-all caches are not shared due to potential - # differences between profiles in _get_implicit_iuse. - self.caches = { - "arch": {}, - "arch_xmatch": {}, - "shared_xmatch": {"cp-list": {}}, - } - - self.include_arches = None - if self.options.include_arches: - self.include_arches = set() - self.include_arches.update( - *[x.split() for x in self.options.include_arches] - ) - self.include_profiles = None - if self.options.include_profiles: - self.include_profiles = set() - self.include_profiles.update( - *[x.split() for x in self.options.include_profiles] - ) - - # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and - # running `svn status` in every package dir will be too expensive. - checks["ebuild_notadded"] = not ( - self.vcs_settings.vcs == "svn" - and self.repolevel < 3 - and self.options.mode != "commit" - ) - - self.effective_scanlist = scanlist - if self.options.if_modified == "y": - self.effective_scanlist = sorted( - vcs_files_to_cps( - chain(self.changed.changed, self.changed.new, self.changed.removed), - self.repo_settings.repodir, - self.repolevel, - self.reposplit, - self.categories, - ) - ) - - # Create our kwargs dict here to initialize the plugins with - self.kwargs = { - "repo_settings": self.repo_settings, - "portdb": self.portdb, - "qatracker": self.qatracker, - "vcs_settings": self.vcs_settings, - "options": self.options, - "metadata_xsd": get_metadata_xsd(self.repo_settings), - "uselist": uselist, - "checks": checks, - "repo_metadata": self.repo_metadata, - "profiles": profiles, - "include_arches": self.include_arches, - "include_profiles": self.include_profiles, - "caches": self.caches, - "repoman_incrementals": self.repoman_incrementals, - "env": self.env, - "have": self.have, - "dev_keywords": self.dev_keywords, - "linechecks": self.moduleconfig.linechecks, - } - # initialize the plugin checks here - self.modules = {} - self._ext_futures = {} - self.pkg_level_futures = None - - def set_kwargs(self, mod): - """Creates a limited set of kwargs to pass to the module's __init__() - - @param mod: module name string - @returns: dictionary - """ - kwargs = {} - for key in self.moduleconfig.controller.modules[mod]["mod_kwargs"]: - kwargs[key] = self.kwargs[key] - return kwargs - - def set_func_kwargs(self, mod, dynamic_data=None): - """Updates the dynamic_data dictionary with any new key, value pairs. - Creates a limited set of kwargs to pass to the modulefunctions to run - - @param mod: module name string - @param dynamic_data: dictionary structure - @returns: dictionary - """ - func_kwargs = self.moduleconfig.controller.modules[mod]["func_kwargs"] - # determine new keys - required = set(list(func_kwargs)) - exist = set(list(dynamic_data)) - new = required.difference(exist) - # update dynamic_data with initialized entries - for key in new: - logging.debug("set_func_kwargs(); adding: %s, %s", key, func_kwargs[key]) - if func_kwargs[key][0] in ["Future", "ExtendedFuture"]: - if key not in self._ext_futures: - logging.debug( - "Adding a new key: %s to the ExtendedFuture dict", key - ) - self._ext_futures[key] = func_kwargs[key] - self._set_future(dynamic_data, key, func_kwargs[key]) - else: # builtin python data type - dynamic_data[key] = DATA_TYPES[func_kwargs[key][0]]() - kwargs = {} - for key in required: - kwargs[key] = dynamic_data[key] - return kwargs - - def reset_futures(self, dynamic_data): - """Reset any Future data types - - @param dynamic_data: dictionary - """ - for key in list(self._ext_futures): - if key not in self.pkg_level_futures: - self._set_future(dynamic_data, key, self._ext_futures[key]) - - @staticmethod - def _set_future(dynamic_data, key, data): - """Set a dynamic_data key to a new ExtendedFuture instance - - @param dynamic_data: dictionary - @param key: tuple of (dictionary-key, default-value) - """ - if data[0] in ["Future", "ExtendedFuture"]: - if data[1] in ["UNSET"]: - dynamic_data[key] = ExtendedFuture() - else: - if data[1] in DATA_TYPES: - default = DATA_TYPES[data[1]]() - else: - default = data[1] - dynamic_data[key] = ExtendedFuture(default) - - def scan_pkgs(self, can_force): - for xpkg in self.effective_scanlist: - xpkg_continue = False - # ebuilds and digests added to cvs respectively. - logging.info("checking package %s", xpkg) - # save memory by discarding xmatch caches from previous package(s) - self.caches["arch_xmatch"].clear() - catdir, pkgdir = xpkg.split("/") - checkdir = self.repo_settings.repodir + "/" + xpkg - checkdir_relative = "" - if self.repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if self.repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - - # Run the status check - if self.kwargs["checks"]["ebuild_notadded"]: - self.vcs_settings.status.check(checkdir, checkdir_relative, xpkg) - - if self.generate_manifest: - if not manifest.Manifest(**self.kwargs).update_manifest(checkdir): - self.qatracker.add_error( - "manifest.bad", os.path.join(xpkg, "Manifest") - ) - if self.options.mode == "manifest": - continue - checkdirlist = os.listdir(checkdir) - - dynamic_data = { - "changelog_modified": False, - "checkdirlist": ExtendedFuture(checkdirlist), - "checkdir": checkdir, - "xpkg": xpkg, - "changed": self.changed, - "checkdir_relative": checkdir_relative, - "can_force": can_force, - "repolevel": self.repolevel, - "catdir": catdir, - "pkgdir": pkgdir, - "validity_future": ExtendedFuture(True), - "y_ebuild": None, - # this needs to be reset at the pkg level only, - # easiest is to just initialize it here - "muselist": ExtendedFuture(set()), - "src_uri_error": ExtendedFuture(), - } - self.pkg_level_futures = [ - "checkdirlist", - "muselist", - "pkgs", - "src_uri_error", - "validity_future", - ] - # need to set it up for ==> self.modules or some other ordered list - logging.debug("***** starting pkgs_loop: %s", self.moduleconfig.pkgs_loop) - for mod in self.moduleconfig.pkgs_loop: - mod_class = self.moduleconfig.controller.get_class(mod) - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) - logging.debug("scan_pkgs; module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInPkgs - if do_it: - for func in functions: - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - xpkg_continue = True - break - - if xpkg_continue: - continue - - # Sort ebuilds in ascending order for the KEYWORDS.dropped check. - pkgs = dynamic_data["pkgs"].get() - ebuildlist = sorted(pkgs.values()) - ebuildlist = [pkg.pf for pkg in ebuildlist] - - if self.kwargs["checks"]["changelog"] and "ChangeLog" not in checkdirlist: - self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - dynamic_data["changelog_modified"] = ( - changelog_path in self.changed.changelogs - ) - - self._scan_ebuilds(ebuildlist, dynamic_data) - - def _scan_ebuilds(self, ebuildlist, dynamic_data): - - for y_ebuild in ebuildlist: - self.reset_futures(dynamic_data) - dynamic_data["y_ebuild"] = y_ebuild - - # initialize per ebuild plugin checks here - # need to set it up for ==> self.modules_list or some other ordered list - for mod in self.moduleconfig.ebuilds_loop: - if mod: - mod_class = self.moduleconfig.controller.get_class(mod) - if mod_class.__name__ not in self.modules: - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class( - **self.set_kwargs(mod) - ) - logging.debug("scan_ebuilds: module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInEbuilds - logging.debug( - "do_it: %s, functions: %s", do_it, [x.__name__ for x in functions] - ) - if do_it: - for func in functions: - logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - # logging.debug("\t>>> Continuing") - break - - logging.debug("Finished ebuild plugin loop, continuing...") - - # Final checks - # initialize per pkg plugin final checks here - # need to set it up for ==> self.modules_list or some other ordered list - for mod in self.moduleconfig.final_loop: - if mod: - mod_class = self.moduleconfig.controller.get_class(mod) - if mod_class.__name__ not in self.modules: - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) - logging.debug("scan_ebuilds final checks: module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInFinal - logging.debug( - "do_it: %s, functions: %s", do_it, [x.__name__ for x in functions] - ) - if do_it: - for func in functions: - logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - # logging.debug("\t>>> Continuing") - break diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev deleted file mode 100644 index a6752fd30bb..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev +++ /dev/null @@ -1,37 +0,0 @@ -This is a revocation certificate for the OpenPGP key: - -pub rsa4096 2020-07-14 [S] - 06B3A311BD775C280D22A9305D90EA06352177F6 -uid Gentoo Portage Test Trusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org> - -A revocation certificate is a kind of "kill switch" to publicly -declare that a key shall not anymore be used. It is not possible -to retract such a revocation certificate once it has been published. - -Use it to revoke this key in case of a compromise or loss of -the secret key. However, if the secret key is still accessible, -it is better to generate a new revocation certificate and give -a reason for the revocation. For details see the description of -of the gpg command "--generate-revocation" in the GnuPG manual. - -To avoid an accidental use of this file, a colon has been inserted -before the 5 dashes below. Remove this colon with a text editor -before importing and publishing this revocation certificate. - -:-----BEGIN PGP PUBLIC KEY BLOCK----- -Comment: This is a revocation certificate - -iQI2BCABCAAgFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAl8OFTwCHQAACgkQXZDq -BjUhd/aXCA/+OgzosMDaDe5DNwkSi2yKdC2X18v8JcaYnXBUR93nXA0LVN7iVWkR -WEH3NuVspQZ5vK+3AHTKabqZFC/buA5oQOH01Ncd4lQISfOOhFiBn5DIPX31BVT0 -iPmVkcxHAD4031ptP4oat6EFclT13SRchtlnAO04JofeHnzQIw3SozQGzXpAA1g4 -BogQ0HWA88HzuEYYE+e/yzZL4D496X1DTaXksg0Py5c4SS6u5pND6lcUtAGxAwa9 -sJFPs+coeURaRV99CrJfdh4u2OkvINTfrKOS6NFBQq6HVH5mLsRXZlcE4Oo4d+fN -XoPrTZnRUqpJADUdjHFvO/lr0fArJTS5IQCVBNFeCMlvgmUPeKWJ1r6Uiwe/UHor -9OP/tK97EqpsaXmHbo0jOUkn5iiUwy784+JBSSu/Q2NxqcBr74aaRdfxvs62dmv7 -droCDQi3ebqTdnlDSaeCIWHyVlSroOhZ+ZETVy193K1X7VXFX3hYKiJ3G8QZwy3e -AlsVGjIHWfC+K+enIn+uwSUvOWPN3upK8kqMRuXvAOppFCE4sTqNbxUnHHXaqo/r -s1q6zVsWVILBk97BHlJph2IaqhV7iIgPU97/r4U/BT11VqDFdVSHcXcs4PDNs5vh -6qttaDiyDqZjwMr+0iDoouHxFpqY8e+3M2gycUgGr2XV6ML0pXE6BqA= -=nIjC ------END PGP PUBLIC KEY BLOCK----- diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev deleted file mode 100644 index 456e0aa5041..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev +++ /dev/null @@ -1,37 +0,0 @@ -This is a revocation certificate for the OpenPGP key: - -pub rsa4096 2020-07-14 [S] - 8DEDA2CDED49C8809287B89D8812797DDF1DD192 -uid Gentoo Portage Test Untrusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org> - -A revocation certificate is a kind of "kill switch" to publicly -declare that a key shall not anymore be used. It is not possible -to retract such a revocation certificate once it has been published. - -Use it to revoke this key in case of a compromise or loss of -the secret key. However, if the secret key is still accessible, -it is better to generate a new revocation certificate and give -a reason for the revocation. For details see the description of -of the gpg command "--generate-revocation" in the GnuPG manual. - -To avoid an accidental use of this file, a colon has been inserted -before the 5 dashes below. Remove this colon with a text editor -before importing and publishing this revocation certificate. - -:-----BEGIN PGP PUBLIC KEY BLOCK----- -Comment: This is a revocation certificate - -iQI2BCABCAAgFiEEje2ize1JyICSh7idiBJ5fd8d0ZIFAl8OFXUCHQAACgkQiBJ5 -fd8d0ZKdwxAAhmkC0V+OLyOU9PCV6ogD9/3b3nVqNIreoc+gxHTLmEvxiMSItqmq -DkcW9RJKAduA/HiLZQ8Yzxw+ldC6kuWqYEjNpSM54VDkrgOePi8W1bVDTCoSp7bo -0JOG4frieqIxA6lhAA2UppH7EPRXoODPLYqooNxWAs3xxVrR6eGAb5l8NXzrymvN -acFfOZ0s5FgADQskQHWVq6TaJn9DrcZxd+b+plSwPYDXqzTChKQ5jw7uMAPUvDkG -JUWgoKiKSrK64bslUq8aEDEZQ4uxjyEi6G0vO/wPL/ysGhS7KkPgCZsEfNjWjajb -jAsdvl1raoHxK/O7llMNr9uRAZtC56pJ//SRDc3kylZrkAo0RNoXQFowT739HWei -2UkCFDfz488VKKrOI8TzTyUvLFEo14ZAXGg1wdHaGnbYMzxpKjP15alOFo6fKIcS -Kz1f/Mab4wf4Sg0XAjQ9pnai1/U9ZF3/NSnRtYgJkLCrIEtRLrgSHJsLDPxjCfGV -jWszAbIk167aA0yKsSmuwkpc5bZqqBaTo904r857fxyt5Les6SOHsV7iNXt7F+am -03Y6u6m2eROba7M67l115vTyYcw5EZVp5j0nI81PXsC9X2DD1ci5xrNmPyEeupC4 -7y7mcGbUYPJAJHJ0kHG4ZYLnNMl42ZYr1ssEeasDwUsLWgVqvx9RkKI= -=kVUQ ------END PGP PUBLIC KEY BLOCK----- diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key Binary files differdeleted file mode 100644 index 0bd1026ad24..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key +++ /dev/null diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key Binary files differdeleted file mode 100644 index 8e29ef43ca5..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key +++ /dev/null diff --git a/repoman/lib/repoman/tests/.gnupg/pubring.kbx b/repoman/lib/repoman/tests/.gnupg/pubring.kbx Binary files differdeleted file mode 100644 index f6367f83bc9..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/pubring.kbx +++ /dev/null diff --git a/repoman/lib/repoman/tests/.gnupg/trustdb.gpg b/repoman/lib/repoman/tests/.gnupg/trustdb.gpg Binary files differdeleted file mode 100644 index db5b1023b67..00000000000 --- a/repoman/lib/repoman/tests/.gnupg/trustdb.gpg +++ /dev/null diff --git a/repoman/lib/repoman/tests/__init__.py b/repoman/lib/repoman/tests/__init__.py deleted file mode 100644 index ba0872c5b5b..00000000000 --- a/repoman/lib/repoman/tests/__init__.py +++ /dev/null @@ -1,328 +0,0 @@ -# tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import argparse -import sys -import time -import unittest - -from unittest.runner import TextTestResult as _TextTestResult - -import repoman -from repoman import REPOMAN_BASE_PATH -from repoman._portage import portage - -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage.const import EPREFIX, GLOBAL_CONFIG_PATH, PORTAGE_BIN_PATH - -if repoman._not_installed: - cnf_path = os.path.join(REPOMAN_BASE_PATH, "cnf") - cnf_path_repoman = cnf_path - cnf_etc_path = cnf_path - cnf_bindir = os.path.join(REPOMAN_BASE_PATH, "bin") - cnf_sbindir = cnf_bindir -else: - cnf_path = os.path.join(EPREFIX or "/", GLOBAL_CONFIG_PATH) - cnf_path_repoman = os.path.join( - EPREFIX or "/", sys.prefix.lstrip(os.sep), "share", "repoman" - ) - cnf_etc_path = os.path.join(EPREFIX or "/", "etc") - cnf_eprefix = EPREFIX - cnf_bindir = os.path.join(EPREFIX or "/", "usr", "bin") - cnf_sbindir = os.path.join(EPREFIX or "/", "usr", "sbin") - - -def main(): - suite = unittest.TestSuite() - basedir = os.path.dirname(os.path.realpath(__file__)) - - usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) - parser = argparse.ArgumentParser(usage=usage) - parser.add_argument( - "-l", "--list", help="list all tests", action="store_true", dest="list_tests" - ) - options, args = parser.parse_known_args(args=sys.argv) - - if ( - os.environ.get("NOCOLOR") in ("yes", "true") - or os.environ.get("TERM") == "dumb" - or not sys.stdout.isatty() - ): - portage.output.nocolor() - - if options.list_tests: - testdir = os.path.dirname(sys.argv[0]) - for mydir in getTestDirs(basedir): - testsubdir = os.path.basename(mydir) - for name in getTestNames(mydir): - print("%s/%s/%s.py" % (testdir, testsubdir, name)) - return os.EX_OK - - if len(args) > 1: - suite.addTests(getTestFromCommandLine(args[1:], basedir)) - else: - for mydir in getTestDirs(basedir): - suite.addTests(getTests(os.path.join(basedir, mydir), basedir)) - - result = TextTestRunner(verbosity=2).run(suite) - if not result.wasSuccessful(): - return 1 - return os.EX_OK - - -def my_import(name): - mod = __import__(name) - components = name.split(".") - for comp in components[1:]: - mod = getattr(mod, comp) - return mod - - -def getTestFromCommandLine(args, base_path): - result = [] - for arg in args: - realpath = os.path.realpath(arg) - path = os.path.dirname(realpath) - f = realpath[len(path) + 1 :] - - if not f.startswith("test") or not f.endswith(".py"): - raise Exception("Invalid argument: '%s'" % arg) - - mymodule = f[:-3] - result.extend(getTestsFromFiles(path, base_path, [mymodule])) - return result - - -def getTestDirs(base_path): - TEST_FILE = b"__test__.py" - testDirs = [] - - # the os.walk help mentions relative paths as being quirky - # I was tired of adding dirs to the list, so now we add __test__.py - # to each dir we want tested. - for root, dirs, files in os.walk(base_path): - try: - root = _unicode_decode(root, encoding=_encodings["fs"], errors="strict") - except UnicodeDecodeError: - continue - - if TEST_FILE in files: - testDirs.append(root) - - testDirs.sort() - return testDirs - - -def getTestNames(path): - files = os.listdir(path) - files = [f[:-3] for f in files if f.startswith("test") and f.endswith(".py")] - files.sort() - return files - - -def getTestsFromFiles(path, base_path, files): - parent_path = path[len(base_path) + 1 :] - parent_module = ".".join(("repoman", "tests", parent_path)) - parent_module = parent_module.replace("/", ".") - result = [] - for mymodule in files: - # Make the trailing / a . for module importing - modname = ".".join((parent_module, mymodule)) - mod = my_import(modname) - result.append(unittest.TestLoader().loadTestsFromModule(mod)) - return result - - -def getTests(path, base_path): - """ - - path is the path to a given subdir ( 'portage/' for example) - This does a simple filter on files in that dir to give us modules - to import - - """ - return getTestsFromFiles(path, base_path, getTestNames(path)) - - -class TextTestResult(_TextTestResult): - """ - We need a subclass of unittest.runner.TextTestResult to handle tests with TODO - - This just adds an addTodo method that can be used to add tests - that are marked TODO; these can be displayed later - by the test runner. - """ - - def __init__(self, stream, descriptions, verbosity): - super(TextTestResult, self).__init__(stream, descriptions, verbosity) - self.todoed = [] - self.portage_skipped = [] - - def addTodo(self, test, info): - self.todoed.append((test, info)) - if self.showAll: - self.stream.writeln("TODO") - elif self.dots: - self.stream.write(".") - - def addPortageSkip(self, test, info): - self.portage_skipped.append((test, info)) - if self.showAll: - self.stream.writeln("SKIP") - elif self.dots: - self.stream.write(".") - - def printErrors(self): - if self.dots or self.showAll: - self.stream.writeln() - self.printErrorList("ERROR", self.errors) - self.printErrorList("FAIL", self.failures) - self.printErrorList("TODO", self.todoed) - self.printErrorList("SKIP", self.portage_skipped) - - -class TestCase(unittest.TestCase): - """ - We need a way to mark a unit test as "ok to fail" - This way someone can add a broken test and mark it as failed - and then fix the code later. This may not be a great approach - (broken code!!??!11oneone) but it does happen at times. - """ - - def __init__(self, *pargs, **kwargs): - unittest.TestCase.__init__(self, *pargs, **kwargs) - self.todo = False - self.portage_skip = None - self.cnf_path = cnf_path - self.cnf_etc_path = cnf_etc_path - self.bindir = cnf_bindir - self.sbindir = cnf_sbindir - - def defaultTestResult(self): - return TextTestResult() - - def run(self, result=None): - if result is None: - result = self.defaultTestResult() - result.startTest(self) - testMethod = getattr(self, self._testMethodName) - try: - try: - self.setUp() - except SystemExit: - raise - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - - ok = False - try: - testMethod() - ok = True - except unittest.SkipTest as e: - result.addPortageSkip(self, "%s: SKIP: %s" % (testMethod, str(e))) - except self.failureException: - if self.portage_skip is not None: - if self.portage_skip is True: - result.addPortageSkip(self, "%s: SKIP" % testMethod) - else: - result.addPortageSkip( - self, "%s: SKIP: %s" % (testMethod, self.portage_skip) - ) - elif self.todo: - result.addTodo(self, "%s: TODO" % testMethod) - else: - result.addFailure(self, sys.exc_info()) - except (KeyboardInterrupt, SystemExit): - raise - except: - result.addError(self, sys.exc_info()) - - try: - self.tearDown() - except SystemExit: - raise - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - ok = False - if ok: - result.addSuccess(self) - finally: - result.stopTest(self) - - def assertRaisesMsg(self, msg, excClass, callableObj, *args, **kwargs): - """Fail unless an exception of class excClass is thrown - by callableObj when invoked with arguments args and keyword - arguments kwargs. If a different type of exception is - thrown, it will not be caught, and the test case will be - deemed to have suffered an error, exactly as for an - unexpected exception. - """ - try: - callableObj(*args, **kwargs) - except excClass: - return - else: - if hasattr(excClass, "__name__"): - excName = excClass.__name__ - else: - excName = str(excClass) - raise self.failureException("%s not raised: %s" % (excName, msg)) - - def assertNotExists(self, path): - """Make sure |path| does not exist""" - if os.path.exists(path): - raise self.failureException("path exists when it should not: %s" % path) - - -class TextTestRunner(unittest.TextTestRunner): - """ - We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable - """ - - def _makeResult(self): - return TextTestResult(self.stream, self.descriptions, self.verbosity) - - def run(self, test): - """ - Run the given test case or test suite. - """ - result = self._makeResult() - startTime = time.time() - test(result) - stopTime = time.time() - timeTaken = stopTime - startTime - result.printErrors() - self.stream.writeln(result.separator2) - run = result.testsRun - self.stream.writeln( - "Ran %d test%s in %.3fs" % (run, run != 1 and "s" or "", timeTaken) - ) - self.stream.writeln() - if not result.wasSuccessful(): - self.stream.write("FAILED (") - failed = len(result.failures) - errored = len(result.errors) - if failed: - self.stream.write("failures=%d" % failed) - if errored: - if failed: - self.stream.write(", ") - self.stream.write("errors=%d" % errored) - self.stream.writeln(")") - else: - self.stream.writeln("OK") - return result - - -test_cps = ["sys-apps/portage", "virtual/portage"] -test_versions = ["1.0", "1.0-r1", "2.3_p4", "1.0_alpha57"] -test_slots = [None, "1", "gentoo-sources-2.6.17", "spankywashere"] -test_usedeps = ["foo", "-bar", ("foo", "bar"), ("foo", "-bar"), ("foo?", "!bar?")] diff --git a/repoman/lib/repoman/tests/__test__.py b/repoman/lib/repoman/tests/__test__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/tests/__test__.py +++ /dev/null diff --git a/repoman/lib/repoman/tests/changelog/__init__.py b/repoman/lib/repoman/tests/changelog/__init__.py deleted file mode 100644 index 532918b6a39..00000000000 --- a/repoman/lib/repoman/tests/changelog/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/lib/repoman/tests/changelog/__test__.py b/repoman/lib/repoman/tests/changelog/__test__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/tests/changelog/__test__.py +++ /dev/null diff --git a/repoman/lib/repoman/tests/changelog/test_echangelog.py b/repoman/lib/repoman/tests/changelog/test_echangelog.py deleted file mode 100644 index f0c43e20b8f..00000000000 --- a/repoman/lib/repoman/tests/changelog/test_echangelog.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import tempfile -import time - -from portage import os -from portage import shutil -from portage.tests import TestCase -from repoman.utilities import UpdateChangeLog - - -class RepomanEchangelogTestCase(TestCase): - def setUp(self): - super(RepomanEchangelogTestCase, self).setUp() - - self.tmpdir = tempfile.mkdtemp(prefix="repoman.echangelog.") - - self.skel_changelog = os.path.join(self.tmpdir, "skel.ChangeLog") - skel = [ - "# ChangeLog for <CATEGORY>/<PACKAGE_NAME>\n", - "# Copyright 1999-2000 Gentoo Foundation; Distributed under the GPL v2\n", - "# $Header: $\n", - ] - self._writelines(self.skel_changelog, skel) - - self.cat = "mycat" - self.pkg = "mypkg" - self.pkgdir = os.path.join(self.tmpdir, self.cat, self.pkg) - os.makedirs(self.pkgdir) - - self.header_pkg = "# ChangeLog for %s/%s\n" % (self.cat, self.pkg) - self.header_copyright = ( - "# Copyright 1999-%s Gentoo Authors; Distributed under the GPL v2\n" - % time.strftime("%Y", time.gmtime()) - ) - self.header_cvs = "# $Header: $\n" - - self.changelog = os.path.join(self.pkgdir, "ChangeLog") - - self.user = "Testing User <portage@gentoo.org>" - - def tearDown(self): - super(RepomanEchangelogTestCase, self).tearDown() - shutil.rmtree(self.tmpdir) - - def _readlines(self, file): - with open(file, "r") as f: - return f.readlines() - - def _writelines(self, file, data): - with open(file, "w") as f: - f.writelines(data) - - def testRejectRootUser(self): - self.assertEqual( - UpdateChangeLog( - self.pkgdir, "me <root@gentoo.org>", "", "", "", "", quiet=True - ), - None, - ) - - def testMissingSkelFile(self): - # Test missing ChangeLog, but with empty skel (i.e. do nothing). - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - "/does/not/exist", - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertTrue(len(actual_cl[0]) > 0) - - def testEmptyChangeLog(self): - # Make sure we do the right thing with a 0-byte ChangeLog - open(self.changelog, "w").close() - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - self.assertEqual(actual_cl[1], self.header_copyright) - self.assertEqual(actual_cl[2], self.header_cvs) - - def testCopyrightUpdate(self): - # Make sure updating the copyright line works - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[1], self.header_copyright) - - def testSkelHeader(self): - # Test skel.ChangeLog -> ChangeLog - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - self.assertNotEqual(actual_cl[-1], "\n") - - def testExistingGoodHeader(self): - # Test existing ChangeLog (correct values) - self._writelines(self.changelog, [self.header_pkg]) - - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - - def testExistingBadHeader(self): - # Test existing ChangeLog (wrong values) - self._writelines(self.changelog, ["# ChangeLog for \n"]) - - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - - def testTrailingNewlines(self): - # Make sure trailing newlines get chomped. - self._writelines(self.changelog, ["#\n", "foo\n", "\n", "bar\n", "\n", "\n"]) - - UpdateChangeLog( - self.pkgdir, - self.user, - "test!", - self.skel_changelog, - self.cat, - self.pkg, - quiet=True, - ) - actual_cl = self._readlines(self.changelog) - self.assertNotEqual(actual_cl[-1], "\n") diff --git a/repoman/lib/repoman/tests/commit/__init__.py b/repoman/lib/repoman/tests/commit/__init__.py deleted file mode 100644 index d74fd94a7d8..00000000000 --- a/repoman/lib/repoman/tests/commit/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/lib/repoman/tests/commit/__test__.py b/repoman/lib/repoman/tests/commit/__test__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/tests/commit/__test__.py +++ /dev/null diff --git a/repoman/lib/repoman/tests/commit/test_commitmsg.py b/repoman/lib/repoman/tests/commit/test_commitmsg.py deleted file mode 100644 index 392d17cb32f..00000000000 --- a/repoman/lib/repoman/tests/commit/test_commitmsg.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2011-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from repoman.actions import Actions -from repoman.tests import TestCase - - -class CommitMessageVerificationTest(TestCase): - def assertGood(self, commitmsg): - res, expl = Actions.verify_commit_message(commitmsg) - self.assertTrue( - res, - """Commit message verification failed for: -%s - -Error: -%s""" - % (commitmsg, expl), - ) - - def assertBad(self, commitmsg, reason_re): - res, expl = Actions.verify_commit_message(commitmsg) - self.assertFalse( - res, - """Commit message verification succeeded unexpectedly, for: -%s - -Expected: /%s/""" - % (commitmsg, reason_re), - ) - self.assertNotIn( - "\n", - expl.strip(), - """Commit message verification returned multiple errors (one expected): -%s - -Expected: /%s/ -Errors: -%s""" - % (commitmsg, reason_re, expl), - ) - ( - self.assertRegex - if hasattr(self, "assertRegex") - else self.assertRegexpMatches - )( - expl, - reason_re, - """Commit message verification did not return expected error, for: -%s - -Expected: /%s/ -Errors: -%s""" - % (commitmsg, reason_re, expl), - ) - - def test_summary_only(self): - self.assertGood("dev-foo/bar: Actually good commit message") - - def test_summary_and_body(self): - self.assertGood( - """dev-foo/bar: Good commit message - -Extended description goes here and is properly wrapped at 72 characters -which is very nice and blah blah. - -Another paragraph for the sake of having one.""" - ) - - def test_summary_and_footer(self): - self.assertGood( - """dev-foo/bar: Good commit message - -Closes: https://bugs.gentoo.org/NNNNNN""" - ) - - def test_summary_body_and_footer(self): - self.assertGood( - """dev-foo/bar: Good commit message - -Extended description goes here and is properly wrapped at 72 characters -which is very nice and blah blah. - -Another paragraph for the sake of having one. - -Closes: https://bugs.gentoo.org/NNNNNN""" - ) - - def test_summary_without_unit_name(self): - self.assertBad("Version bump", r"summary.*logical unit name") - - def test_multiline_summary(self): - self.assertBad( - """dev-foo/bar: Commit message with very long summary -that got wrapped because of length""", - r"single.*line.*summary", - ) - - def test_overlong_summary(self): - self.assertBad( - "dev-foo/bar: Commit message with very long summary \ -in a single line that should trigger an explicit error", - r"summary.*too long", - ) - - def test_summary_with_very_long_package_name(self): - self.assertGood( - "dev-foo/foo-bar-bar-baz-bar-bar-foo-bar-bar-\ -baz-foo-baz-baz-foo: We do not fail because pkgname was long" - ) - - def test_multiple_footers(self): - self.assertBad( - """dev-foo/bar: Good summary - -Bug: https://bugs.gentoo.org/NNNNNN - -Closes: https://github.com/gentoo/gentoo/pull/NNNN""", - r"multiple footer", - ) - - def test_gentoo_bug(self): - self.assertBad( - """dev-foo/bar: Good summary - -Gentoo-Bug: NNNNNN""", - r"Gentoo-Bug", - ) - - def test_bug_with_number(self): - self.assertBad( - """dev-foo/bar: Good summary - -Bug: NNNNNN""", - r"Bug.*full URL", - ) - - def test_closes_with_number(self): - self.assertBad( - """dev-foo/bar: Good summary - -Closes: NNNNNN""", - r"Closes.*full URL", - ) - - def test_body_too_long(self): - self.assertBad( - """dev-foo/bar: Good summary - -But the body is not wrapped properly and has very long lines that are \ -very hard to read and blah blah blah -blah blah.""", - r"body.*wrapped", - ) diff --git a/repoman/lib/repoman/tests/runTests.py b/repoman/lib/repoman/tests/runTests.py deleted file mode 100644 index 4a081551ea7..00000000000 --- a/repoman/lib/repoman/tests/runTests.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# runTests.py -- Portage Unit Test Functionality -# Copyright 2006-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import os -import os.path as osp -import sys -import grp -import platform -import pwd -import signal -import tempfile -from distutils.dir_util import copy_tree - - -def debug_signal(signum, frame): - import pdb - - pdb.set_trace() - - -if platform.python_implementation() == "Jython": - debug_signum = signal.SIGUSR2 # bug #424259 -else: - debug_signum = signal.SIGUSR1 - -signal.signal(debug_signum, debug_signal) - -# Pretend that the current user's uid/gid are the 'portage' uid/gid, -# so things go smoothly regardless of the current user and global -# user/group configuration. -os.environ["PORTAGE_USERNAME"] = pwd.getpwuid(os.getuid()).pw_name -os.environ["PORTAGE_GRPNAME"] = grp.getgrgid(os.getgid()).gr_name - -# Insert our parent dir so we can do shiny import "tests" -# This line courtesy of Marienz and Pkgcore ;) -repoman_pym = osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__)))) -sys.path.insert(0, repoman_pym) - -# Add in the parent portage python modules -portage_pym = osp.dirname(osp.dirname(repoman_pym)) + "/lib" -sys.path.insert(0, portage_pym) - -# import our centrally initialized portage instance -from repoman._portage import portage - -portage._internal_caller = True -import repoman.tests as tests - -# Ensure that we don't instantiate portage.settings, so that tests should -# work the same regardless of global configuration file state/existence. -portage._disable_legacy_globals() -from portage.util._eventloop.global_event_loop import global_event_loop -from portage.const import PORTAGE_BIN_PATH - -if os.environ.get("NOCOLOR") in ("yes", "true"): - portage.output.nocolor() - -path = os.environ.get("PATH", "").split(":") -path = [x for x in path if x] - -insert_bin_path = True -try: - insert_bin_path = not path or not os.path.samefile(path[0], PORTAGE_BIN_PATH) -except OSError: - pass - -if insert_bin_path: - path.insert(0, PORTAGE_BIN_PATH) - os.environ["PATH"] = ":".join(path) - -# Copy GPG test keys to temporary directory -gpg_path = tempfile.mkdtemp(prefix="gpg_") - -copy_tree(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".gnupg"), gpg_path) - -os.chmod(gpg_path, 0o700) -os.environ["PORTAGE_GNUPGHOME"] = gpg_path - -if __name__ == "__main__": - try: - sys.exit(tests.main()) - finally: - global_event_loop().close() diff --git a/repoman/lib/repoman/tests/simple/__init__.py b/repoman/lib/repoman/tests/simple/__init__.py deleted file mode 100644 index 532918b6a39..00000000000 --- a/repoman/lib/repoman/tests/simple/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/lib/repoman/tests/simple/__test__.py b/repoman/lib/repoman/tests/simple/__test__.py deleted file mode 100644 index e69de29bb2d..00000000000 --- a/repoman/lib/repoman/tests/simple/__test__.py +++ /dev/null diff --git a/repoman/lib/repoman/tests/simple/test_simple.py b/repoman/lib/repoman/tests/simple/test_simple.py deleted file mode 100644 index 41735b8b426..00000000000 --- a/repoman/lib/repoman/tests/simple/test_simple.py +++ /dev/null @@ -1,512 +0,0 @@ -# Copyright 2011-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import collections -import subprocess -import sys -import time -import types - -from repoman._portage import portage -from portage import os -from portage.process import find_binary -from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from portage.util import ensure_dirs -from portage.util.futures import asyncio -from portage.util.futures._asyncio.streams import _reader -from portage.util._async.AsyncFunction import AsyncFunction - -# pylint: disable=ungrouped-imports -from repoman import REPOMAN_BASE_PATH -from repoman.copyrights import update_copyright_year -from repoman.main import _repoman_init, _repoman_scan, _handle_result -from repoman.tests import TestCase - - -class RepomanRun(types.SimpleNamespace): - async def run(self): - self.expected = getattr(self, "expected", None) or {"returncode": 0} - if self.debug: - fd_pipes = {} - pr = None - pw = None - else: - pr, pw = os.pipe() - fd_pipes = {1: pw, 2: pw} - pr = open(pr, "rb", 0) - - proc = AsyncFunction( - scheduler=asyncio.get_event_loop(), - target=self._subprocess, - args=(self.args, self.cwd, self.env, self.expected, self.debug), - fd_pipes=fd_pipes, - ) - - proc.start() - if pw is not None: - os.close(pw) - - await proc.async_wait() - - if pr is None: - stdio = None - else: - stdio = await _reader(pr) - - self.result = { - "stdio": stdio, - "result": proc.result, - } - - @staticmethod - def _subprocess(args, cwd, env, expected, debug): - os.chdir(cwd) - os.environ.update(env) - portage.const.EPREFIX = env["PORTAGE_OVERRIDE_EPREFIX"] - if debug: - args = ["-vvvv"] + args - repoman_vars = _repoman_init(["repoman"] + args) - if repoman_vars.exitcode is not None: - return {"returncode": repoman_vars.exitcode} - result = _repoman_scan(*repoman_vars) - returncode = _handle_result(*repoman_vars, result) - qawarnings = repoman_vars.vcs_settings.qatracker.qawarnings - warns = collections.defaultdict(list) - fails = collections.defaultdict(list) - for qacat, issues in repoman_vars.vcs_settings.qatracker.fails.items(): - if qacat in qawarnings: - warns[qacat].extend(issues) - else: - fails[qacat].extend(issues) - result = {"returncode": returncode} - if fails: - result["fails"] = fails - if warns: - result["warns"] = warns - return result - - -class SimpleRepomanTestCase(TestCase): - def testCopyrightUpdate(self): - test_cases = ( - ( - "2011", - "# Copyright 1999-2008 Gentoo Foundation; Distributed under the GPL v2", - "# Copyright 1999-2011 Gentoo Authors; Distributed under the GPL v2", - ), - ( - "2011", - "# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2", - "# Copyright 1999-2011 Gentoo Authors; Distributed under the GPL v2", - ), - ( - "1999", - "# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2", - "# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2", - ), - ( - "2018", - "# Copyright 1999-2008 Gentoo Authors; Distributed under the GPL v2", - "# Copyright 1999-2018 Gentoo Authors; Distributed under the GPL v2", - ), - ( - "2018", - "# Copyright 2017 Gentoo Authors; Distributed under the GPL v2", - "# Copyright 2017-2018 Gentoo Authors; Distributed under the GPL v2", - ), - ) - - for year, before, after in test_cases: - self.assertEqual(update_copyright_year(year, before), after) - - def _must_skip(self): - xmllint = find_binary("xmllint") - if not xmllint: - return "xmllint not found" - - try: - __import__("xml.etree.ElementTree") - __import__("xml.parsers.expat").parsers.expat.ExpatError - except (AttributeError, ImportError): - return "python is missing xml support" - - def testSimple(self): - debug = False - - skip_reason = self._must_skip() - if skip_reason: - self.portage_skip = skip_reason - self.assertFalse(True, skip_reason) - return - - copyright_header = ( - """# Copyright 1999-%s Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -""" - % time.gmtime().tm_year - ) - - pkg_preinst_references_forbidden_var = """ -pkg_preinst() { - echo "This ${A} reference is not allowed. Neither is this $BROOT reference." -} -""" - - repo_configs = { - "test_repo": { - "layout.conf": ("update-changelog = true",), - } - } - - profiles = ( - ("x86", "default/linux/x86/test_profile", "stable"), - ("x86", "default/linux/x86/test_dev", "dev"), - ("x86", "default/linux/x86/test_exp", "exp"), - ) - - profile = {"eapi": ("5",), "package.use.stable.mask": ("dev-libs/A flag",)} - - ebuilds = { - "dev-libs/A-0": { - "COPYRIGHT_HEADER": copyright_header, - "DESCRIPTION": "Desc goes here", - "EAPI": "5", - "HOMEPAGE": "https://example.com", - "IUSE": "flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/B[flag] )", - }, - "dev-libs/A-1": { - "COPYRIGHT_HEADER": copyright_header, - "DESCRIPTION": "Desc goes here", - "EAPI": "4", - "HOMEPAGE": "https://example.com", - "IUSE": "flag", - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/B[flag] )", - }, - "dev-libs/B-1": { - "COPYRIGHT_HEADER": copyright_header, - "DESCRIPTION": "Desc goes here", - "EAPI": "4", - "HOMEPAGE": "https://example.com", - "IUSE": "flag", - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - }, - "dev-libs/C-0": { - "COPYRIGHT_HEADER": copyright_header, - "DESCRIPTION": "Desc goes here", - "EAPI": "7", - "HOMEPAGE": "https://example.com", - "IUSE": "flag", - # must be unstable, since dev-libs/A[flag] is stable masked - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/A[flag] )", - "MISC_CONTENT": pkg_preinst_references_forbidden_var, - }, - } - licenses = ["GPL-2"] - arch_list = ["x86"] - metadata_xsd = os.path.join(REPOMAN_BASE_PATH, "cnf/metadata.xsd") - metadata_xml_files = ( - ( - "dev-libs/A", - { - "flags": "<flag name='flag' restrict='>=dev-libs/A-0'>Description of how USE='flag' affects this package</flag>", - }, - ), - ( - "dev-libs/B", - { - "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ( - "dev-libs/C", - { - "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ) - - use_desc = (("flag", "Description of how USE='flag' affects packages"),) - - playground = ResolverPlayground( - ebuilds=ebuilds, profile=profile, repo_configs=repo_configs, debug=debug - ) - - loop = asyncio._wrap_loop() - loop.run_until_complete( - asyncio.ensure_future( - self._async_test_simple( - playground, - metadata_xml_files, - profiles, - profile, - licenses, - arch_list, - use_desc, - metadata_xsd, - copyright_header, - debug, - ), - loop=loop, - ) - ) - - async def _async_test_simple( - self, - playground, - metadata_xml_files, - profiles, - profile, - licenses, - arch_list, - use_desc, - metadata_xsd, - copyright_header, - debug, - ): - settings = playground.settings - eprefix = settings["EPREFIX"] - eroot = settings["EROOT"] - portdb = playground.trees[playground.eroot]["porttree"].dbapi - homedir = os.path.join(eroot, "home") - distdir = os.path.join(eprefix, "distdir") - test_repo_location = settings.repositories["test_repo"].location - profiles_dir = os.path.join(test_repo_location, "profiles") - license_dir = os.path.join(test_repo_location, "licenses") - - repoman_cmd = ( - portage._python_interpreter, - "-b", - "-Wd", - os.path.join(self.bindir, "repoman"), - ) - - git_binary = find_binary("git") - git_cmd = (git_binary,) - - cp_binary = find_binary("cp") - self.assertEqual(cp_binary is None, False, "cp command not found") - cp_cmd = (cp_binary,) - - test_ebuild = portdb.findname("dev-libs/A-1") - self.assertFalse(test_ebuild is None) - - committer_name = "Gentoo Dev" - committer_email = "gentoo-dev@gentoo.org" - expected_warnings = { - "returncode": 0, - "warns": { - "variable.phase": [ - "dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable A: Forbidden reference to variable specified by PMS", - "dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable BROOT: Forbidden reference to variable specified by PMS", - ] - }, - } - - git_test = ( - ("", RepomanRun(args=["--version"])), - ("", RepomanRun(args=["manifest"])), - ( - "", - git_cmd - + ( - "config", - "--global", - "user.name", - committer_name, - ), - ), - ( - "", - git_cmd - + ( - "config", - "--global", - "user.email", - committer_email, - ), - ), - ("", git_cmd + ("init-db",)), - ("", git_cmd + ("add", ".")), - ("", git_cmd + ("commit", "-a", "-m", "add whole repo")), - ("", RepomanRun(args=["full", "-d"], expected=expected_warnings)), - ( - "", - RepomanRun( - args=[ - "full", - "--include-profiles", - "default/linux/x86/test_profile", - ], - expected=expected_warnings, - ), - ), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")), - ( - "", - RepomanRun( - args=["commit", "-m", "cat/pkg: bump to version 2"], - expected=expected_warnings, - ), - ), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "3.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "3.ebuild")), - ( - "dev-libs", - RepomanRun( - args=["commit", "-m", "cat/pkg: bump to version 3"], - expected=expected_warnings, - ), - ), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "4.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "4.ebuild")), - ( - "dev-libs/A", - RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 4"]), - ), - ) - - env = { - "PORTAGE_OVERRIDE_EPREFIX": eprefix, - "DISTDIR": distdir, - "GENTOO_COMMITTER_NAME": committer_name, - "GENTOO_COMMITTER_EMAIL": committer_email, - "HOME": homedir, - "PATH": os.environ["PATH"], - "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"], - "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"], - "PORTAGE_REPOSITORIES": settings.repositories.config_string(), - "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), - } - - if os.environ.get("SANDBOX_ON") == "1": - # avoid problems from nested sandbox instances - env["FEATURES"] = "-sandbox -usersandbox" - - dirs = [homedir, license_dir, profiles_dir, distdir] - try: - for d in dirs: - ensure_dirs(d) - with open(os.path.join(test_repo_location, "skel.ChangeLog"), "w") as f: - f.write(copyright_header) - with open(os.path.join(profiles_dir, "profiles.desc"), "w") as f: - for x in profiles: - f.write("%s %s %s\n" % x) - - # ResolverPlayground only created the first profile, - # so create the remaining ones. - for x in profiles[1:]: - sub_profile_dir = os.path.join(profiles_dir, x[1]) - ensure_dirs(sub_profile_dir) - for config_file, lines in profile.items(): - file_name = os.path.join(sub_profile_dir, config_file) - with open(file_name, "w") as f: - for line in lines: - f.write("%s\n" % line) - - for x in licenses: - open(os.path.join(license_dir, x), "wb").close() - with open(os.path.join(profiles_dir, "arch.list"), "w") as f: - for x in arch_list: - f.write("%s\n" % x) - with open(os.path.join(profiles_dir, "use.desc"), "w") as f: - for k, v in use_desc: - f.write("%s - %s\n" % (k, v)) - for cp, xml_data in metadata_xml_files: - with open( - os.path.join(test_repo_location, cp, "metadata.xml"), "w" - ) as f: - f.write(playground.metadata_xml_template % xml_data) - # Use a symlink to test_repo, in order to trigger bugs - # involving canonical vs. non-canonical paths. - test_repo_symlink = os.path.join(eroot, "test_repo_symlink") - os.symlink(test_repo_location, test_repo_symlink) - metadata_xsd_dest = os.path.join( - test_repo_location, "metadata/xml-schema/metadata.xsd" - ) - os.makedirs(os.path.dirname(metadata_xsd_dest)) - os.symlink(metadata_xsd, metadata_xsd_dest) - - if debug: - # The subprocess inherits both stdout and stderr, for - # debugging purposes. - stdout = None - else: - # The subprocess inherits stderr so that any warnings - # triggered by python -Wd will be visible. - stdout = subprocess.PIPE - - for cwd in ("", "dev-libs", "dev-libs/A", "dev-libs/B", "dev-libs/C"): - abs_cwd = os.path.join(test_repo_symlink, cwd) - - proc = await asyncio.create_subprocess_exec( - *(repoman_cmd + ("full",)), - env=env, - stderr=None, - stdout=stdout, - cwd=abs_cwd - ) - - if debug: - await proc.wait() - else: - output, _err = await proc.communicate() - await proc.wait() - if proc.returncode != os.EX_OK: - portage.writemsg(output) - - self.assertEqual( - os.EX_OK, proc.returncode, "repoman failed in %s" % (cwd,) - ) - - if git_binary is not None: - for cwd, cmd in git_test: - abs_cwd = os.path.join(test_repo_symlink, cwd) - if isinstance(cmd, RepomanRun): - cmd.cwd = abs_cwd - cmd.env = env - cmd.debug = debug - await cmd.run() - if cmd.result["result"] != cmd.expected and cmd.result.get( - "stdio" - ): - portage.writemsg(cmd.result["stdio"]) - try: - self.assertEqual(cmd.result["result"], cmd.expected) - except Exception: - print(cmd.result["result"], file=sys.stderr, flush=True) - raise - continue - - proc = await asyncio.create_subprocess_exec( - *cmd, env=env, stderr=None, stdout=stdout, cwd=abs_cwd - ) - - if debug: - await proc.wait() - else: - output, _err = await proc.communicate() - await proc.wait() - if proc.returncode != os.EX_OK: - portage.writemsg(output) - - self.assertEqual( - os.EX_OK, - proc.returncode, - "%s failed in %s" - % ( - cmd, - cwd, - ), - ) - finally: - playground.cleanup() diff --git a/repoman/lib/repoman/utilities.py b/repoman/lib/repoman/utilities.py deleted file mode 100644 index 6c541447041..00000000000 --- a/repoman/lib/repoman/utilities.py +++ /dev/null @@ -1,590 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: Utilities -# Copyright 2007-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains utility functions to help repoman find ebuilds to -scan""" - -__all__ = [ - "editor_is_executable", - "FindPackagesToScan", - "FindPortdir", - "get_commit_message_with_editor", - "get_committer_name", - "have_ebuild_dir", - "have_profile_dir", - "UpdateChangeLog", -] - -import errno -import io -from itertools import chain -import logging -import pwd -import stat -import time -import textwrap -import difflib -import tempfile - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage import shutil -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -from portage import util -from portage.localization import _ -from portage.process import find_binary -from portage.output import green - -# pylint: disable=ungrouped-imports -from repoman.copyrights import update_copyright_year - - -normalize_path = util.normalize_path -util.initialize_logger() - - -def have_profile_dir(path, maxdepth=3, filename="profiles.desc"): - """ - Try to figure out if 'path' has a profiles/ - dir in it by checking for the given filename. - """ - while path != "/" and maxdepth: - if os.path.exists(os.path.join(path, "profiles", filename)): - return normalize_path(path) - path = normalize_path(path + "/..") - maxdepth -= 1 - - -def have_ebuild_dir(path, maxdepth=3): - """ - Try to figure out if 'path' or a subdirectory contains one or more - ebuild files named appropriately for their parent directory. - """ - stack = [(normalize_path(path), 1)] - while stack: - path, depth = stack.pop() - basename = os.path.basename(path) - try: - listdir = os.listdir(path) - except OSError: - continue - for filename in listdir: - abs_filename = os.path.join(path, filename) - try: - st = os.stat(abs_filename) - except OSError: - continue - if stat.S_ISDIR(st.st_mode): - if depth < maxdepth: - stack.append((abs_filename, depth + 1)) - elif stat.S_ISREG(st.st_mode): - if filename.endswith(".ebuild") and filename.startswith(basename + "-"): - return os.path.dirname(os.path.dirname(path)) - - -def FindPackagesToScan(settings, startdir, reposplit): - """Try to find packages that need to be scanned - - Args: - settings - portage.config instance, preferably repoman_settings - startdir - directory that repoman was run in - reposplit - root of the repository - Returns: - A list of directories to scan - """ - - def AddPackagesInDir(path): - """Given a list of dirs, add any packages in it""" - ret = [] - pkgdirs = os.listdir(path) - for d in pkgdirs: - if d == "CVS" or d.startswith("."): - continue - p = os.path.join(path, d) - - if os.path.isdir(p): - cat_pkg_dir = os.path.join(*p.split(os.path.sep)[-2:]) - logging.debug("adding %s to scanlist" % cat_pkg_dir) - ret.append(cat_pkg_dir) - return ret - - scanlist = [] - repolevel = len(reposplit) - if repolevel == 1: # root of the tree, startdir = repodir - for cat in settings.categories: - path = os.path.join(startdir, cat) - if not os.path.isdir(path): - continue - scanlist.extend(AddPackagesInDir(path)) - elif repolevel == 2: # category level, startdir = catdir - # We only want 1 segment of the directory, - # this is why we use catdir instead of startdir. - catdir = reposplit[-2] - if catdir not in settings.categories: - logging.warn( - "%s is not a valid category according to profiles/categories, " - "skipping checks in %s" % (catdir, catdir) - ) - else: - scanlist = AddPackagesInDir(catdir) - elif repolevel == 3: # pkgdir level, startdir = pkgdir - catdir = reposplit[-2] - pkgdir = reposplit[-1] - if catdir not in settings.categories: - logging.warn( - "%s is not a valid category according to profiles/categories, " - "skipping checks in %s" % (catdir, catdir) - ) - else: - path = os.path.join(catdir, pkgdir) - logging.debug("adding %s to scanlist" % path) - scanlist.append(path) - return scanlist - - -def editor_is_executable(editor): - """ - Given an EDITOR string, validate that it refers to - an executable. This uses shlex_split() to split the - first component and do a PATH lookup if necessary. - - @param editor: An EDITOR value from the environment. - @type: string - @rtype: bool - @return: True if an executable is found, False otherwise. - """ - editor_split = util.shlex_split(editor) - if not editor_split: - return False - filename = editor_split[0] - if not os.path.isabs(filename): - return find_binary(filename) is not None - return os.access(filename, os.X_OK) and os.path.isfile(filename) - - -def get_commit_message_with_editor(editor, message=None, prefix=""): - """ - Execute editor with a temporary file as it's argument - and return the file content afterwards. - - @param editor: An EDITOR value from the environment - @type: string - @param message: An iterable of lines to show in the editor. - @type: iterable - @param prefix: Suggested prefix for the commit message summary line. - @type: string - @rtype: string or None - @return: A string on success or None if an error occurs. - """ - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - filename = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - try: - with open(filename, "wb") as mymsg: - mymsg.write( - _unicode_encode( - _( - prefix + "\n\n# Please enter the commit message " - "for your changes.\n# (Comment lines starting " - "with '#' will not be included)\n" - ), - encoding=_encodings["content"], - errors="backslashreplace", - ) - ) - if message: - mymsg.write(b"#\n") - for line in message: - mymsg.write( - _unicode_encode( - "#" + line, - encoding=_encodings["content"], - errors="backslashreplace", - ) - ) - retval = os.system(editor + " '%s'" % filename) - if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK): - return None - try: - with io.open( - _unicode_encode(filename, encoding=_encodings["fs"], errors="strict"), - mode="r", - encoding=_encodings["content"], - errors="replace", - ) as f: - mylines = f.readlines() - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - return None - return "".join(line for line in mylines if not line.startswith("#")) - finally: - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - - -def FindPortdir(settings): - """Try to figure out what repo we are in and whether we are in a regular - tree or an overlay. - - Basic logic is: - - 1. Determine what directory we are in (supports symlinks). - 2. Build a list of directories from / to our current location - 3. Iterate over PORTDIR_OVERLAY, if we find a match, - search for a profiles directory in the overlay. If it has one, - make it portdir, otherwise make it portdir_overlay. - 4. If we didn't find an overlay in PORTDIR_OVERLAY, - see if we are in PORTDIR; if so, set portdir_overlay to PORTDIR. - If we aren't in PORTDIR, see if PWD has a profiles dir, if so, - set portdir_overlay and portdir to PWD, else make them False. - 5. If we haven't found portdir_overlay yet, - it means the user is doing something odd, report an error. - 6. If we haven't found a portdir yet, set portdir to PORTDIR. - - Args: - settings - portage.config instance, preferably repoman_settings - Returns: - list(portdir, portdir_overlay, location) - """ - - portdir = None - portdir_overlay = None - location = os.getcwd() - pwd = _unicode_decode(os.environ.get("PWD", ""), encoding=_encodings["fs"]) - if pwd and pwd != location and os.path.realpath(pwd) == location: - # getcwd() returns the canonical path but that makes it hard for repoman to - # orient itself if the user has symlinks in their repository structure. - # We use os.environ["PWD"], if available, to get the non-canonical path of - # the current working directory (from the shell). - location = pwd - - location = normalize_path(location) - - path_ids = {} - p = location - s = None - while True: - s = os.stat(p) - path_ids[(s.st_dev, s.st_ino)] = p - if p == "/": - break - p = os.path.dirname(p) - if location[-1] != "/": - location += "/" - - for overlay in portage.util.shlex_split(settings["PORTDIR_OVERLAY"]): - overlay = os.path.realpath(overlay) - try: - s = os.stat(overlay) - except OSError: - continue - overlay = path_ids.get((s.st_dev, s.st_ino)) - if overlay is None: - continue - if overlay[-1] != "/": - overlay += "/" - if True: - portdir_overlay = overlay - subdir = location[len(overlay) :] - if subdir and subdir[-1] != "/": - subdir += "/" - if have_profile_dir(location, subdir.count("/")): - portdir = portdir_overlay - break - - # Couldn't match location with anything from PORTDIR_OVERLAY, - # so fall back to have_profile_dir() checks alone. Assume that - # an overlay will contain at least a "repo_name" file while a - # master repo (portdir) will contain at least a "profiles.desc" - # file. - if not portdir_overlay: - portdir_overlay = have_profile_dir(location, filename="repo_name") - if not portdir_overlay: - portdir_overlay = have_ebuild_dir(location) - if portdir_overlay: - subdir = location[len(portdir_overlay) :] - if subdir and subdir[-1] != os.sep: - subdir += os.sep - if have_profile_dir(location, subdir.count(os.sep)): - portdir = portdir_overlay - - if not portdir_overlay: - if (settings["PORTDIR"] + os.path.sep).startswith(location): - portdir_overlay = settings["PORTDIR"] - else: - portdir_overlay = have_profile_dir(location) - portdir = portdir_overlay - - if not portdir_overlay: - msg = ( - "Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY" - + " from the current working directory" - ) - logging.critical(msg) - return (None, None, None) - - if not portdir: - portdir = settings["PORTDIR"] - - if not portdir_overlay.endswith("/"): - portdir_overlay += "/" - - if not portdir.endswith("/"): - portdir += "/" - - return [normalize_path(x) for x in (portdir, portdir_overlay, location)] - - -def get_committer_name(env=None): - """Generate a committer string like echangelog does.""" - if env is None: - env = os.environ - if "GENTOO_COMMITTER_NAME" in env and "GENTOO_COMMITTER_EMAIL" in env: - user = "%s <%s>" % (env["GENTOO_COMMITTER_NAME"], env["GENTOO_COMMITTER_EMAIL"]) - elif "GENTOO_AUTHOR_NAME" in env and "GENTOO_AUTHOR_EMAIL" in env: - user = "%s <%s>" % (env["GENTOO_AUTHOR_NAME"], env["GENTOO_AUTHOR_EMAIL"]) - elif "ECHANGELOG_USER" in env: - user = env["ECHANGELOG_USER"] - else: - pwd_struct = pwd.getpwuid(os.getuid()) - gecos = pwd_struct.pw_gecos.split(",")[0] # bug #80011 - user = "%s <%s@gentoo.org>" % (gecos, pwd_struct.pw_name) - return user - - -def UpdateChangeLog( - pkgdir, - user, - msg, - skel_path, - category, - package, - new=(), - removed=(), - changed=(), - pretend=False, - quiet=False, -): - """ - Write an entry to an existing ChangeLog, or create a new one. - Updates copyright year on changed files, and updates the header of - ChangeLog with the contents of skel.ChangeLog. - """ - - if "<root@" in user: - if not quiet: - logging.critical("Please set ECHANGELOG_USER or run as non-root") - return None - - # ChangeLog times are in UTC - gmtime = time.gmtime() - year = time.strftime("%Y", gmtime) - date = time.strftime("%d %b %Y", gmtime) - - cl_path = os.path.join(pkgdir, "ChangeLog") - clold_lines = [] - clnew_lines = [] - old_header_lines = [] - header_lines = [] - - clold_file = None - try: - clold_file = io.open( - _unicode_encode(cl_path, encoding=_encodings["fs"], errors="strict"), - mode="r", - encoding=_encodings["repo.content"], - errors="replace", - ) - except EnvironmentError: - pass - - f, clnew_path = tempfile.mkstemp() - - # construct correct header first - try: - if clold_file is not None: - # retain header from old ChangeLog - first_line = True - for line in clold_file: - line_strip = line.strip() - if line_strip and line[:1] != "#": - clold_lines.append(line) - break - # always make sure cat/pkg is up-to-date in case we are - # moving packages around, or copied from another pkg, or ... - if first_line: - if line.startswith("# ChangeLog for"): - line = "# ChangeLog for %s/%s\n" % (category, package) - first_line = False - old_header_lines.append(line) - header_lines.append(update_copyright_year(year, line)) - if not line_strip: - break - - clskel_file = None - if not header_lines: - # delay opening this until we find we need a header - try: - clskel_file = io.open( - _unicode_encode( - skel_path, encoding=_encodings["fs"], errors="strict" - ), - mode="r", - encoding=_encodings["repo.content"], - errors="replace", - ) - except EnvironmentError: - pass - - if clskel_file is not None: - # read skel.ChangeLog up to first empty line - for line in clskel_file: - line_strip = line.strip() - if not line_strip: - break - line = line.replace("<CATEGORY>", category) - line = line.replace("<PACKAGE_NAME>", package) - line = update_copyright_year(year, line) - header_lines.append(line) - header_lines.append("\n") - clskel_file.close() - - # write new ChangeLog entry - clnew_lines.extend(header_lines) - newebuild = False - for fn in new: - if not fn.endswith(".ebuild"): - continue - ebuild = fn.split(os.sep)[-1][0:-7] - clnew_lines.append("*%s (%s)\n" % (ebuild, date)) - newebuild = True - if newebuild: - clnew_lines.append("\n") - trivial_files = ("ChangeLog", "Manifest") - display_new = ["+" + elem for elem in new if elem not in trivial_files] - display_removed = ["-" + elem for elem in removed] - display_changed = [elem for elem in changed if elem not in trivial_files] - if not (display_new or display_removed or display_changed): - # If there's nothing else to display, show one of the - # trivial files. - for fn in trivial_files: - if fn in new: - display_new = ["+" + fn] - break - elif fn in changed: - display_changed = [fn] - break - - display_new.sort() - display_removed.sort() - display_changed.sort() - - mesg = "%s; %s %s:" % ( - date, - user, - ", ".join(chain(display_new, display_removed, display_changed)), - ) - for line in textwrap.wrap( - mesg, - 80, - initial_indent=" ", - subsequent_indent=" ", - break_on_hyphens=False, - ): - clnew_lines.append("%s\n" % line) - for line in textwrap.wrap(msg, 80, initial_indent=" ", subsequent_indent=" "): - clnew_lines.append("%s\n" % line) - # Don't append a trailing newline if the file is new. - if clold_file is not None: - clnew_lines.append("\n") - - f = io.open( - f, mode="w", encoding=_encodings["repo.content"], errors="backslashreplace" - ) - - for line in clnew_lines: - f.write(line) - - # append stuff from old ChangeLog - if clold_file is not None: - - if clold_lines: - # clold_lines may contain a saved non-header line - # that we want to write first. - # Also, append this line to clnew_lines so that the - # unified_diff call doesn't show it as removed. - for line in clold_lines: - f.write(line) - clnew_lines.append(line) - - else: - # ensure that there is no more than one blank - # line after our new entry - for line in clold_file: - if line.strip(): - f.write(line) - break - - # Now prepend old_header_lines to clold_lines, for use - # in the unified_diff call below. - clold_lines = old_header_lines + clold_lines - - # Trim any trailing newlines. - lines = clold_file.readlines() - clold_file.close() - while lines and lines[-1] == "\n": - del lines[-1] - f.writelines(lines) - f.close() - - # show diff - if not quiet: - for line in difflib.unified_diff( - clold_lines, clnew_lines, fromfile=cl_path, tofile=cl_path, n=0 - ): - util.writemsg_stdout(line, noiselevel=-1) - util.writemsg_stdout("\n", noiselevel=-1) - - if pretend: - # remove what we've done - os.remove(clnew_path) - else: - # rename to ChangeLog, and set permissions - try: - clold_stat = os.stat(cl_path) - except OSError: - clold_stat = None - - shutil.move(clnew_path, cl_path) - - if clold_stat is None: - util.apply_permissions(cl_path, mode=0o644) - else: - util.apply_stat_permissions(cl_path, clold_stat) - - if clold_file is None: - return True - else: - return False - except IOError as e: - err = "Repoman is unable to create/write to Changelog.new file: %s" % (e,) - logging.critical(err) - # try to remove if possible - try: - os.remove(clnew_path) - except OSError: - pass - return None - - -def repoman_sez(msg): - print(green("RepoMan sez:"), msg) diff --git a/repoman/man/repoman.1 b/repoman/man/repoman.1 deleted file mode 100644 index 5dbc41560e0..00000000000 --- a/repoman/man/repoman.1 +++ /dev/null @@ -1,478 +0,0 @@ -.TH "REPOMAN" "1" "March 2021" "Repoman VERSION" "Repoman" -.SH NAME -repoman \- Gentoo's program to enforce a minimal level of quality assurance in -packages added to the ebuild repository -.SH SYNOPSIS -\fBrepoman\fR [\fIoption\fR] [\fImode\fR] -.SH DESCRIPTION -.BR "Quality is job zero." - -.BR repoman -checks the quality of ebuild repositories. - -Note: \fBrepoman commit\fR only works \fIinside local\fR cvs, git, or -subversion repositories. - -Note: Messages pertaining to specific lines may be inaccurate in the -prescence of continuation lines from use of the \fI\\\fR character in -BASH. -.SH OPTIONS -.TP -\fB-a\fR, \fB--ask\fR -Request a confirmation before commiting -.TP -\fB-b\fR, \fB--bug\fR -Include a bug reference in the commit message footer. The argument can -be either a Gentoo bug number or a full bug URL (either Gentoo -or upstream). Gentoo bug URLs are automatically shortened to -the canonical \fBhttps://bugs.gentoo.org/NNNNNN\fR form, and HTTPS -is forced for known bug trackers. - -When pushing to the Gentoo repository, the reference to the commit -will be automatically added as a comment on the bug. -.TP -\fB-c\fR, \fB--closes\fR -Include a \fBCloses\fR tag in the commit message footer that can be used -to close bugs and pull requests. The argument can be either a Gentoo bug -number or a full PR/bug URL. Gentoo bug URLs are automatically shortened -to the canonical \fBhttps://bugs.gentoo.org/NNNNNN\fR form, and HTTPS -is forced for known bug trackers. - -When pushing to the Gentoo repository, the referenced bugs will be -closed as RESOLVED/FIXED automatically with a comment referencing -the commit. Furthermore, GitHub pull requests will be closed as well -due to the automatic GitHub mirroring. - -Other platforms using the \fBCloses\fR tag include GitLab and Bitbucket. -.TP -\fB\-\-digest=<y|n>\fR -Automatically update Manifest digests for modified files. This -option triggers a behavior that is very similar to that enabled -by FEATURES="digest" in \fBmake.conf\fR(5). In order to enable -this behavior by default for repoman alone, add -\fB\-\-digest=y\fR to the \fIREPOMAN_DEFAULT_OPTS\fR variable in -\fBmake.conf\fR(5). The \fBmanifest\-check\fR mode will -automatically ignore the \-\-digest option. - -\fBNOTE:\fR -This option does not trigger update of digests for Manifest DIST -entries that already exist. Replacement of existing Manifest -DIST entries can be forced by using the \fBmanifest\fR mode -together with the \fB\-\-force\fR option. -.TP -\fB-f\fR, \fB--force\fR -Force commit to proceed, regardless of QA issues. For convenience, this option -causes the most time consuming QA checks to be skipped. The commit message will -include an indication that this option has been enabled, together with the -usual portage version stamp. - -When used together with \fBmanifest\fR mode, \fB--force\fR causes existing -digests to be replaced for any files that exist in ${DISTDIR}. -Existing digests are assumed to be correct for files that would otherwise -have to be downloaded in order to recompute digests. \fBWARNING:\fR When -replacing existing digests, it is the user's responsibility to ensure that -files contained in ${DISTDIR} have the correct identities. Especially beware -of partially downloaded files. -.TP -\fB-S\fR, \fB--straight-to-stable\fR -Allow committing straight to stable -.TP -\fB-q\fR, \fB--quiet\fR -Be less verbose about extraneous info -.TP -\fB-p\fR, \fB--pretend\fR -Don't commit or fix anything; just show what would be done -.TP -\fB\-j\fR, \fB\-\-jobs\fR -Specifies the number of jobs (processes) to run simultaneously. -.TP -\fB\-l\fR, \fB\-\-load-average\fR -Specifies that no new jobs (processes) should be started if there are others -jobs running and the load average is at least load (a floating\-point number). -.TP -\fB-x\fR, \fB--xmlparse\fR -Forces the metadata.xml parse check to be carried out -.TP -\fB-v\fR, \fB--verbose\fR -Displays every package name while checking -.TP -\fB\-\-echangelog=<y|n|force>\fR -For commit mode, call echangelog if ChangeLog is unmodified (or -regardless of modification if 'force' is specified). This option -can be enabled by default for a particular repository by setting -"update\-changelog = true" in metadata/layout.conf (see -\fBportage\fR(5)). -.TP -\fB\-\-experimental\-inherit=<y|n>\fR -Enable experimental inherit.missing checks which may misbehave when the -internal eclass database becomes outdated. -.TP -\fB\-\-experimental\-repository\-modules=<y|n>\fR -Enable experimental repository modules: -\fIhttps://wiki.gentoo.org/wiki/Project:Portage/Repoman-Module-specs\fR -.TP -\fB\-\-if\-modified=<y|n>\fR -Only check packages that have uncommitted modifications -.TP -\fB\-i\fR, \fB\-\-ignore\-arches\fR -Ignore arch-specific failures (where arch != host) -.TP -\fB\-\-ignore\-default\-opts\fR -Do not use the \fIREPOMAN_DEFAULT_OPTS\fR environment variable. -.TP -\fB\-I\fR, \fB\-\-ignore\-masked\fR -Ignore masked packages (not allowed with commit mode) -.TP -.BR "\-\-include\-arches " ARCHES -A space separated list of arches used to filter the selection of -profiles for dependency checks. -.TP -.BR "\-\-include\-profiles " PROFILES -A space separated list of profiles used to -define the selection of profiles for dependency checks. -.TP -\fB\-d\fR, \fB\-\-include\-dev\fR -Include dev profiles in dependency checks. -.TP -\fB\-e <y|n>\fR, \fB\-\-include\-exp\-profiles=<y|n>\fR -Include exp profiles in dependency checks. -.TP -\fB\-\-unmatched\-removal\fR -Enable strict checking of package.mask and package.unmask files for -unmatched removal atoms. -.TP -\fB\-\-without\-mask\fR -Behave as if no package.mask entries exist (not allowed with commit mode) -.TP -\fB-m\fR, \fB--commitmsg\fR -Adds a commit message via the command line -.TP -\fB-M\fR, \fB--commitmsgfile\fR -Adds a commit message from the specified file. This option also will perform -an automatic text substitution of a leading "cat/pkg: " string (upper or lower -case) with the actual category/package prefix as defined by the required message -format. Use this option for templating a common commit message for multiple -package updates. -.TP -\fB-V\fR, \fB--version\fR -Show version info -.TP -\fB-h\fR, \fB--help\fR -Show this screen -.SH MODES -.TP -.B full -Scan directory tree for QA issues (full listing) -.TP -.B help -Show this screen -.TP -.B scan -Scan directory tree for QA issues (short listing) -.TP -.B fix -Fix simple QA issues (stray digests, missing digests) -.TP -.B manifest -Generate a Manifest (fetches distfiles if necessary). See the \fB\-\-force\fR -option if you would like to replace existing distfiles digests. -.TP -.B manifest-check -Check Manifests for missing or incorrect digests -.TP -.B commit -Scan directory tree for QA issues; if OK, commit via VCS -.TP -.B ci -Synonym for commit -.SH QA KEYWORDS -.TP -.B CVS/Entries.IO_error -Attempting to commit, and an IO error was encountered access the Entries file -.TP -.B DESCRIPTION.missing -Ebuilds that have a missing or empty DESCRIPTION variable -.TP -.B EAPI.definition -EAPI definition does not conform to PMS section 7.3.1 (first -non\-comment, non\-blank line). See bug #402167. -.TP -.B EAPI.deprecated -Ebuilds that use features that are deprecated in the current EAPI -.TP -.B EAPI.incompatible -Ebuilds that use features that are only available with a different EAPI -.TP -.B EAPI.unsupported -Ebuilds that have an unsupported EAPI version (you must upgrade portage) -.TP -.B HOMEPAGE.missing -Ebuilds that have a missing or empty HOMEPAGE variable -.TP -.B HOMEPAGE.missingurischeme -HOMEPAGE is missing an URI scheme -.TP -.B HOMEPAGE.virtual -Virtuals that have a non-empty HOMEPAGE variable -.TP -.B IUSE.invalid -This ebuild has a variable in IUSE that is not in the use.desc or its -metadata.xml file -.TP -.B IUSE.missing -This ebuild has a USE conditional which references a flag that is not listed in -IUSE -.TP -.B KEYWORDS.dropped -Ebuilds that appear to have dropped KEYWORDS for some arch -.TP -.B KEYWORDS.invalid -This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for -which no valid profile was found -.TP -.B KEYWORDS.missing -Ebuilds that have a missing or empty KEYWORDS variable -.TP -.B KEYWORDS.stable -Ebuilds that have been added directly with stable KEYWORDS -.TP -.B KEYWORDS.stupid -Ebuilds that use KEYWORDS=-* instead of package.mask -.TP -.B KEYWORDS.unsorted -Ebuilds that contain KEYWORDS which are not sorted alphabetically. -.TP -.B LICENSE.deprecated -This ebuild is listing a deprecated license. -.TP -.B LICENSE.invalid -This ebuild is listing a license that doesnt exist in portages license/ dir. -.TP -.B LICENSE.missing -Ebuilds that have a missing or empty LICENSE variable -.TP -.B LICENSE.syntax -Syntax error in LICENSE (usually an extra/missing space/parenthesis) -.TP -.B LICENSE.virtual -Virtuals that have a non-empty LICENSE variable -.TP -.B LIVEVCS.stable -Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout with stable -keywords. -.TP -.B LIVEVCS.unmasked -Ebuild is a live ebuild (cvs, git, darcs, svn, etc) checkout but has keywords -and is not masked in the global package.mask. -.TP -.B PDEPEND.suspect -PDEPEND contains a package that usually only belongs in DEPEND -.TP -.B RDEPEND.implicit -RDEPEND is unset in the ebuild which triggers implicit RDEPEND=$DEPEND -assignment (prior to EAPI 4) -.TP -.B RDEPEND.suspect -RDEPEND contains a package that usually only belongs in DEPEND -.TP -.B PROPERTIES.syntax -Syntax error in PROPERTIES (usually an extra/missing space/parenthesis) -.TP -.B RESTRICT.syntax -Syntax error in RESTRICT (usually an extra/missing space/parenthesis) -.B SLOT.invalid -Ebuilds that have a missing or invalid SLOT variable value -.TP -.B SRC_URI.mirror -A uri listed in profiles/thirdpartymirrors is found in SRC_URI -.TP -.B changelog.ebuildadded -An ebuild was added but the ChangeLog was not modified -.TP -.B changelog.missing -Missing ChangeLog files -.TP -.B changelog.notadded -ChangeLogs that exist but have not been added to the vcs -.TP -.B dependency.bad -User-visible ebuilds with unsatisfied dependencies (matched against *visible* -ebuilds) -.TP -.B dependency.badindev -User-visible ebuilds with unsatisfied dependencies (matched against *visible* -ebuilds) in developing arch -.TP -.B dependency.badinexp -User-visible ebuilds with unsatisfied dependencies (matched against *visible* -ebuilds) in experimental arch -.TP -.B dependency.badmasked -Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) -.TP -.B dependency.badmaskedindev -Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in -developing arch -.TP -.B dependency.badmaskedinexp -Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in -experimental arch -.TP -.B dependency.badtilde -Uses the ~ dep operator with a non-zero revision part, which is useless (the -revision is ignored) -.TP -.B dependency.deprecated -Ebuild has a dependency that refers to a deprecated package -.TP -.B dependency.syntax -Syntax error in dependency string (usually an extra/missing space/parenthesis) -.TP -.B dependency.unknown -Ebuild has a dependency that refers to an unknown package (which may be -valid if it is a blocker for a renamed/removed package, or is an -alternative choice provided by an overlay) -.TP -.B digest.assumed -Existing digest must be assumed correct (Package level only) -.TP -.B digest.missing -Some files listed in SRC_URI aren't referenced in the Manifest -.TP -.B digest.unused -Some files listed in the Manifest aren't referenced in SRC_URI -.TP -.B ebuild.absdosym -Ebuild uses 'dosym' with explicit absolute path where relative path -could be used -.TP -.B ebuild.badheader -This ebuild has a malformed header -.TP -.B ebuild.invalidname -Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 -versioning extensions) -.TP -.B ebuild.majorsyn -This ebuild has a major syntax error that may cause the ebuild to fail -partially or fully -.TP -.B ebuild.minorsyn -This ebuild has a minor syntax error that contravenes gentoo coding style -.TP -.B ebuild.namenomatch -Ebuild files that do not have the same name as their parent directory -.TP -.B ebuild.nesteddie -Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild. -.TP -.B ebuild.notadded -Ebuilds that exist but have not been added to the vcs -.TP -.B ebuild.output -A simple sourcing of the ebuild produces output; this breaks ebuild policy. -.TP -.B ebuild.patches -PATCHES variable should be a bash array to ensure white space safety -.TP -.B ebuild.syntax -Error generating cache entry for ebuild; typically caused by ebuild syntax -error or digest verification failure. -.TP -.B file.UTF8 -File is not UTF8 compliant -.TP -.B file.empty -Empty file in the files directory -.TP -.B file.executable -Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the -executable bit -.TP -.B file.name -File/dir name must be composed of only the following chars: a-zA-Z0-9._-+: -.TP -.B file.size -Files in the files directory must be under 20 KiB -.TP -.B inherit.missing -Ebuild uses functions from an eclass but does not inherit it -.TP -.B inherit.unused -Ebuild inherits an eclass but does not use it -.TP -.B inherit.deprecated -Ebuild inherits a deprecated eclass -.TP -.B java.eclassesnotused -With virtual/jdk in DEPEND you must inherit a java eclass. Refer to -\fIhttps://wiki.gentoo.org/wiki/Project:Java\fR for more information. -.TP -.B manifest.bad -Manifest has missing or incorrect digests -.TP -.B metadata.bad -Bad metadata.xml files -.TP -.B metadata.missing -Missing metadata.xml files -.TP -.B metadata.warning -Warnings in metadata.xml files -.TP -.B repo.eapi.banned -The ebuild uses an EAPI which is banned by the repository's -metadata/layout.conf settings. -.TP -.B repo.eapi.deprecated -The ebuild uses an EAPI which is deprecated by the repository's -metadata/layout.conf settings. -.TP -.B IUSE.rubydeprecated -The ebuild has set a ruby interpreter in USE_RUBY, that is not available as a ruby target anymore -.TP -.B portage.internal -The ebuild uses an internal Portage function or variable -.TP -.B upstream.workaround -The ebuild works around an upstream bug, an upstream bug should be filed and -tracked in bugs.gentoo.org -.TP -.B usage.obsolete -The ebuild makes use of an obsolete construct -.TP -.B variable.invalidchar -A variable contains an invalid character that is not part of the ASCII -character set. -.TP -.B variable.phase -Variable referenced found within scope of incorrect ebuild phase as specified by PMS. -.TP -.B variable.readonly -Assigning a readonly variable -.TP -.B variable.usedwithhelpers -Ebuild uses D, ROOT, BROOT, ED, EROOT or EPREFIX with helpers -.TP -.B virtual.suspect -Ebuild contains a package that usually should be pulled via virtual/, -not directly. -.TP -.B wxwidgets.eclassnotused -Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass. Refer to -bug #305469 for more information. -.SH "BEHAVIOR" -When invoked from a level higher than a package directory, \fBrepoman\fR -will recurse through a directory tree and execute the given command -on a per\-package basis, e.g. \fBrepoman manifest\fR at the root of -the repository will generate manifests for every package within the -repository. -.SH "REPORTING BUGS" -Please report bugs via https://bugs.gentoo.org/ -.SH AUTHORS -.nf -Daniel Robbins <drobbins@gentoo.org> -Saleem Abdulrasool <compnerd@gentoo.org> -.fi -.SH "SEE ALSO" -.BR emerge (1) diff --git a/repoman/runtests b/repoman/runtests deleted file mode 100755 index f02bfe87555..00000000000 --- a/repoman/runtests +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python -# Copyright 2010-2020 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 -# -# Note: We don't want to import portage modules directly because we do things -# like run the testsuite through multiple versions of python. - -"""Helper script to run portage unittests against different python versions. - -Note: Any additional arguments will be passed down directly to the underlying -unittest runner. This lets you select specific tests to execute. -""" - -import argparse -import os -import shutil -import subprocess -import sys -import tempfile - - -# These are the versions we fully support and require to pass tests. -PYTHON_SUPPORTED_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"] -# The rest are just "nice to have". -PYTHON_NICE_VERSIONS = ["pypy3", "3.10"] - -EPREFIX = os.environ.get("PORTAGE_OVERRIDE_EPREFIX", "/") - - -class Colors(object): - """Simple object holding color constants.""" - - _COLORS_YES = ("y", "yes", "true") - _COLORS_NO = ("n", "no", "false") - - WARN = GOOD = BAD = NORMAL = "" - - def __init__(self, colorize=None): - if colorize is None: - nocolors = os.environ.get("NOCOLOR", "false") - # Ugh, look away, for here we invert the world! - if nocolors in self._COLORS_YES: - colorize = False - elif nocolors in self._COLORS_NO: - colorize = True - else: - raise ValueError("$NOCOLORS is invalid: %s" % nocolors) - else: - if colorize in self._COLORS_YES: - colorize = True - elif colorize in self._COLORS_NO: - colorize = False - else: - raise ValueError("--colors is invalid: %s" % colorize) - - if colorize: - self.WARN = "\033[1;33m" - self.GOOD = "\033[1;32m" - self.BAD = "\033[1;31m" - self.NORMAL = "\033[0m" - - -def get_python_executable(ver): - """Find the right python executable for |ver|""" - if ver in ("pypy", "pypy3"): - prog = ver - else: - prog = "python" + ver - return os.path.join(EPREFIX, "usr", "bin", prog) - - -def get_parser(): - """Return a argument parser for this module""" - epilog = """Examples: -List all the available unittests. -$ %(prog)s --list - -Run against specific versions of python. -$ %(prog)s --python-versions '2.7 3.3' - -Run just one unittest. -$ %(prog)s lib/portage/tests/xpak/test_decodeint.py -""" - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=epilog, - ) - parser.add_argument( - "--keep-temp", - default=False, - action="store_true", - help="Do not delete the temporary directory when exiting", - ) - parser.add_argument( - "--color", - type=str, - default=None, - help="Whether to use colorized output (default is auto)", - ) - parser.add_argument( - "--python-versions", - action="append", - help="Versions of python to test (default is test available)", - ) - return parser - - -def main(argv): - parser = get_parser() - opts, args = parser.parse_known_args(argv) - colors = Colors(colorize=opts.color) - - # Figure out all the versions we want to test. - if opts.python_versions is None: - ignore_missing = True - pyversions = PYTHON_SUPPORTED_VERSIONS + PYTHON_NICE_VERSIONS - else: - ignore_missing = False - pyversions = [] - for ver in opts.python_versions: - if ver == "supported": - pyversions.extend(PYTHON_SUPPORTED_VERSIONS) - else: - pyversions.extend(ver.split()) - - here = os.path.dirname(__file__) - run_path = os.path.join(here, "lib/repoman/tests/runTests.py") - tempdir = None - try: - # Set up a single tempdir for all the tests to use. - # This way we know the tests won't leak things on us. - tempdir = tempfile.mkdtemp(prefix="repoman.runtests.") - os.environ["TMPDIR"] = tempdir - - # Actually test those versions now. - statuses = [] - for ver in pyversions: - prog = get_python_executable(ver) - cmd = [prog, "-b", "-Wd", run_path] + args - if os.access(prog, os.X_OK): - print( - "%sTesting with Python %s...%s" % (colors.GOOD, ver, colors.NORMAL) - ) - statuses.append((ver, subprocess.call(cmd))) - elif not ignore_missing: - print( - "%sCould not find requested Python %s%s" - % (colors.BAD, ver, colors.NORMAL) - ) - statuses.append((ver, 1)) - else: - print("%sSkip Python %s...%s" % (colors.WARN, ver, colors.NORMAL)) - print() - finally: - if tempdir is not None: - if opts.keep_temp: - print("Temporary directory left behind:\n%s" % tempdir) - else: - # Nuke our tempdir and anything that might be under it. - shutil.rmtree(tempdir, True) - - # Then summarize it all. - print("\nSummary:\n") - width = 10 - header = "| %-*s | %s" % (width, "Version", "Status") - print("%s\n|%s" % (header, "-" * (len(header) - 1))) - exit_status = 0 - for ver, status in statuses: - exit_status += status - if status: - color = colors.BAD - msg = "FAIL" - else: - color = colors.GOOD - msg = "PASS" - print( - "| %s%-*s%s | %s%s%s" - % (color, width, ver, colors.NORMAL, color, msg, colors.NORMAL) - ) - exit(exit_status) - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - print("interrupted ...", file=sys.stderr) - exit(1) diff --git a/repoman/setup.py b/repoman/setup.py deleted file mode 100755 index 8231c8f7587..00000000000 --- a/repoman/setup.py +++ /dev/null @@ -1,523 +0,0 @@ -#!/usr/bin/env python -# Copyright 1998-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -try: - from setuptools.core import setup, Command - from setuptools.command.build import build - from setuptools.command.build_scripts import build_scripts - from setuptools.command.clean import clean - from setuptools.command.install import install - from setuptools.command.install_data import install_data - from setuptools.command.install_lib import install_lib - from setuptools.command.install_scripts import install_scripts - from setuptools.command.sdist import sdist - from setuptools.dep_util import newer - from setuptools.dir_util import mkpath, remove_tree, copy_tree - from setuptools.util import change_root, subst_vars -except ImportError: - from distutils.core import setup, Command - from distutils.command.build import build - from distutils.command.build_scripts import build_scripts - from distutils.command.clean import clean - from distutils.command.install import install - from distutils.command.install_data import install_data - from distutils.command.install_lib import install_lib - from distutils.command.install_scripts import install_scripts - from distutils.command.sdist import sdist - from distutils.dep_util import newer - from distutils.dir_util import mkpath, remove_tree, copy_tree - from distutils.util import change_root, subst_vars - -import codecs -import collections -import os -import os.path -import re -import subprocess -import sys - -# change the cwd to this one -os.chdir(os.path.dirname(os.path.realpath(__file__))) - -# TODO: -# - smarter rebuilds of docs w/ 'install_docbook' and 'install_epydoc'. - -x_scripts = { - "bin": [ - "bin/repoman", - ], -} - - -class x_build(build): - """Build command with extra build_man call.""" - - def run(self): - build.run(self) - self.run_command("build_man") - - -class build_man(Command): - """Perform substitutions in manpages.""" - - user_options = [] - - def initialize_options(self): - self.build_base = None - - def finalize_options(self): - self.set_undefined_options("build", ("build_base", "build_base")) - - def run(self): - for d, files in self.distribution.data_files: - if not d.startswith("$mandir/"): - continue - - for source in files: - target = os.path.join(self.build_base, source) - mkpath(os.path.dirname(target)) - - if not newer(source, target) and not newer(__file__, target): - continue - - print("copying and updating %s -> %s" % (source, target)) - - with codecs.open(source, "r", "utf8") as f: - data = f.readlines() - data[0] = data[0].replace("VERSION", self.distribution.get_version()) - with codecs.open(target, "w", "utf8") as f: - f.writelines(data) - - -class x_build_scripts_custom(build_scripts): - def finalize_options(self): - build_scripts.finalize_options(self) - if "dir_name" in dir(self): - self.build_dir = os.path.join(self.build_dir, self.dir_name) - if self.dir_name in x_scripts: - self.scripts = x_scripts[self.dir_name] - else: - self.scripts = set(self.scripts) - for other_files in x_scripts.values(): - self.scripts.difference_update(other_files) - - def run(self): - # group scripts by subdirectory - split_scripts = collections.defaultdict(list) - for f in self.scripts: - dir_name = os.path.dirname(f[len("bin/") :]) - split_scripts[dir_name].append(f) - - base_dir = self.build_dir - base_scripts = self.scripts - for d, files in split_scripts.items(): - self.build_dir = os.path.join(base_dir, d) - self.scripts = files - self.copy_scripts() - - # restore previous values - self.build_dir = base_dir - self.scripts = base_scripts - - -class x_build_scripts_bin(x_build_scripts_custom): - dir_name = "bin" - - -class x_build_scripts(build_scripts): - def initialize_option(self): - build_scripts.initialize_options(self) - - def finalize_options(self): - build_scripts.finalize_options(self) - - def run(self): - self.run_command("build_scripts_bin") - - -class x_clean(clean): - """clean extended for doc & post-test cleaning""" - - def clean_tests(self): - # do not remove incorrect dirs accidentally - top_dir = os.path.normpath(os.path.join(self.build_lib, "..")) - cprefix = os.path.commonprefix((self.build_base, top_dir)) - if cprefix != self.build_base: - return - - bin_dir = os.path.join(top_dir, "bin") - if os.path.exists(bin_dir): - remove_tree(bin_dir) - - conf_dir = os.path.join(top_dir, "cnf") - if os.path.islink(conf_dir): - print("removing %s symlink" % repr(conf_dir)) - os.unlink(conf_dir) - - pni_file = os.path.join(top_dir, ".repoman_not_installed") - if os.path.exists(pni_file): - print("removing %s" % repr(pni_file)) - os.unlink(pni_file) - - def clean_man(self): - man_dir = os.path.join(self.build_base, "man") - if os.path.exists(man_dir): - remove_tree(man_dir) - - def run(self): - if self.all: - self.clean_tests() - self.clean_docs() - self.clean_man() - - clean.run(self) - - -class x_install(install): - """install command with extra Portage paths""" - - user_options = install.user_options + [ - # note: $prefix and $exec_prefix are reserved for Python install - ("system-prefix=", None, "Prefix for architecture-independent data"), - ("system-exec-prefix=", None, "Prefix for architecture-specific data"), - ("bindir=", None, "Install directory for main executables"), - ("datarootdir=", None, "Data install root directory"), - ("docdir=", None, "Documentation install directory"), - ("htmldir=", None, "HTML documentation install directory"), - ("mandir=", None, "Manpage root install directory"), - ("portage-base=", "b", "Portage install base"), - ( - "portage-bindir=", - None, - "Install directory for Portage internal-use executables", - ), - ("portage-datadir=", None, "Install directory for data files"), - ("sbindir=", None, "Install directory for superuser-intended executables"), - ("sysconfdir=", None, "System configuration path"), - ] - - # note: the order is important for proper substitution - paths = [ - ("system_prefix", "/usr"), - ("system_exec_prefix", "$system_prefix"), - ("bindir", "$system_exec_prefix/bin"), - ("sbindir", "$system_exec_prefix/sbin"), - ("sysconfdir", "/etc"), - ("datarootdir", "$system_prefix/share"), - ("docdir", "$datarootdir/doc/$package-$version"), - ("htmldir", "$docdir/html"), - ("mandir", "$datarootdir/man"), - ("portage_base", "$system_exec_prefix/lib/portage"), - ("portage_bindir", "$portage_base/bin"), - ("portage_datadir", "$datarootdir/portage"), - # not customized at the moment - ("logrotatedir", "$sysconfdir/logrotate.d"), - ("portage_confdir", "$portage_datadir/config"), - ("portage_setsdir", "$portage_confdir/sets"), - ] - - def initialize_options(self): - install.initialize_options(self) - - for key, default in self.paths: - setattr(self, key, default) - self.subst_paths = {} - - def finalize_options(self): - install.finalize_options(self) - - # substitute variables - new_paths = { - "package": self.distribution.get_name(), - "version": self.distribution.get_version(), - } - for key, _default in self.paths: - new_paths[key] = subst_vars(getattr(self, key), new_paths) - setattr(self, key, new_paths[key]) - self.subst_paths = new_paths - - -class x_install_data(install_data): - """install_data with customized path support""" - - user_options = install_data.user_options - - def initialize_options(self): - install_data.initialize_options(self) - self.build_base = None - self.paths = None - - def finalize_options(self): - install_data.finalize_options(self) - self.set_undefined_options("build", ("build_base", "build_base")) - self.set_undefined_options("install", ("subst_paths", "paths")) - - def run(self): - self.run_command("build_man") - - def process_data_files(df): - for d, files in df: - # substitute man sources - if d.startswith("$mandir/"): - files = [os.path.join(self.build_base, v) for v in files] - - # substitute variables in path - d = subst_vars(d, self.paths) - yield (d, files) - - old_data_files = self.data_files - self.data_files = process_data_files(self.data_files) - - install_data.run(self) - self.data_files = old_data_files - - -class x_install_lib(install_lib): - """install_lib command with Portage path substitution""" - - user_options = install_lib.user_options - - def initialize_options(self): - install_lib.initialize_options(self) - - def finalize_options(self): - install_lib.finalize_options(self) - self.set_undefined_options( - "install", - ) - - def install(self): - ret = install_lib.install(self) - - def rewrite_file(path, val_dict): - path = os.path.join(self.install_dir, path) - print("Rewriting %s" % path) - with codecs.open(path, "r", "utf-8") as f: - data = f.read() - - for varname, val in val_dict.items(): - regexp = r"(?m)^(%s\s*=).*$" % varname - repl = r"\1 %s" % repr(val) - - data = re.sub(regexp, repl, data) - - with codecs.open(path, "w", "utf-8") as f: - f.write(data) - - rewrite_file( - "repoman/__init__.py", - { - "VERSION": self.distribution.get_version(), - }, - ) - - return ret - - -class x_install_scripts_custom(install_scripts): - def initialize_options(self): - install_scripts.initialize_options(self) - self.root = None - - def finalize_options(self): - self.set_undefined_options( - "install", ("root", "root"), (self.var_name, "install_dir") - ) - install_scripts.finalize_options(self) - self.build_dir = os.path.join(self.build_dir, self.dir_name) - - # prepend root - if self.root is not None: - self.install_dir = change_root(self.root, self.install_dir) - - -class x_install_scripts_bin(x_install_scripts_custom): - dir_name = "bin" - var_name = "bindir" - - -class x_install_scripts(install_scripts): - def initialize_option(self): - pass - - def finalize_options(self): - pass - - def run(self): - self.run_command("install_scripts_bin") - - -class x_sdist(sdist): - """sdist defaulting to .tar.xz format, and archive files owned by root""" - - def initialize_options(self): - super().initialize_options() - self.formats = ["xztar"] - - def finalize_options(self): - if self.owner is None: - self.owner = "root" - if self.group is None: - self.group = "root" - - sdist.finalize_options(self) - - -class build_tests(x_build_scripts_custom): - """Prepare build dir for running tests.""" - - def initialize_options(self): - x_build_scripts_custom.initialize_options(self) - self.build_base = None - self.build_lib = None - - def finalize_options(self): - x_build_scripts_custom.finalize_options(self) - self.set_undefined_options( - "build", ("build_base", "build_base"), ("build_lib", "build_lib") - ) - - # since we will be writing to $build_lib/.., it is important - # that we do not leave $build_base - self.top_dir = os.path.normpath(os.path.join(self.build_lib, "..")) - cprefix = os.path.commonprefix((self.build_base, self.top_dir)) - if cprefix != self.build_base: - raise SystemError("build_lib must be a subdirectory of build_base") - - self.build_dir = os.path.join(self.top_dir, "bin") - - def run(self): - self.run_command("build_py") - - # install all scripts $build_lib/../bin - # (we can't do a symlink since we want shebangs corrected) - x_build_scripts_custom.run(self) - - # symlink 'cnf' directory - conf_dir = os.path.join(self.top_dir, "cnf") - if os.path.exists(conf_dir): - if not os.path.islink(conf_dir): - raise SystemError( - "%s exists and is not a symlink (collision)" % repr(conf_dir) - ) - os.unlink(conf_dir) - conf_src = os.path.relpath("cnf", self.top_dir) - print("Symlinking %s -> %s" % (conf_dir, conf_src)) - os.symlink(conf_src, conf_dir) - - # create $build_lib/../.repoman_not_installed - # to enable proper paths in tests - with open(os.path.join(self.top_dir, ".repoman_not_installed"), "w"): - pass - - -class test(Command): - """run tests""" - - user_options = [] - - def initialize_options(self): - self.build_lib = None - - def finalize_options(self): - self.set_undefined_options("build", ("build_lib", "build_lib")) - - def run(self): - self.run_command("build_tests") - # copy GPG test keys - copy_tree( - os.path.join( - self.build_lib, "..", "..", "lib", "repoman", "tests", ".gnupg" - ), - os.path.join(self.build_lib, "repoman", "tests", ".gnupg"), - ) - os.chmod(os.path.join(self.build_lib, "repoman", "tests", ".gnupg"), 0o700) - subprocess.check_call( - [ - sys.executable, - "-bWd", - os.path.join(self.build_lib, "repoman/tests/runTests.py"), - ] - ) - - -def find_packages(): - for dirpath, _dirnames, filenames in os.walk("lib"): - if "__init__.py" in filenames: - yield os.path.relpath(dirpath, "lib") - - -def find_scripts(): - for dirpath, _dirnames, filenames in os.walk("bin"): - for f in filenames: - if f not in ["deprecated-path"]: - yield os.path.join(dirpath, f) - - -def get_manpages(): - linguas = os.environ.get("LINGUAS") - if linguas is not None: - linguas = linguas.split() - - for dirpath, _dirnames, filenames in os.walk("man"): - groups = collections.defaultdict(list) - for f in filenames: - _fn, suffix = f.rsplit(".", 1) - groups[suffix].append(os.path.join(dirpath, f)) - - topdir = dirpath[len("man/") :] - if not topdir or linguas is None or topdir in linguas: - for g, mans in groups.items(): - yield [os.path.join("$mandir", topdir, "man%s" % g), mans] - - -setup( - name="repoman", - version="3.0.3", - url="https://wiki.gentoo.org/wiki/Project:Portage", - author="Gentoo Portage Development Team", - author_email="dev-portage@gentoo.org", - package_dir={"": "lib"}, - packages=list(find_packages()), - # something to cheat build & install commands - scripts=list(find_scripts()), - data_files=list(get_manpages()) - + [ - ["$docdir", ["NEWS", "RELEASE-NOTES"]], - ["share/repoman/qa_data", ["cnf/qa_data/qa_data.yaml"]], - ["share/repoman/linechecks", ["cnf/linechecks/linechecks.yaml"]], - [ - "share/repoman/repository", - [ - "cnf/repository/linechecks.yaml", - "cnf/repository/qa_data.yaml", - "cnf/repository/repository.yaml", - ], - ], - ], - cmdclass={ - "build": x_build, - "build_man": build_man, - "build_scripts": x_build_scripts, - "build_scripts_bin": x_build_scripts_bin, - "build_tests": build_tests, - "clean": x_clean, - "install": x_install, - "install_data": x_install_data, - "install_lib": x_install_lib, - "install_scripts": x_install_scripts, - "install_scripts_bin": x_install_scripts_bin, - "sdist": x_sdist, - "test": test, - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", - "Operating System :: POSIX", - "Programming Language :: Python :: 3", - "Topic :: System :: Installation/Setup", - ], -) @@ -20,6 +20,5 @@ deps = setenv = PYTHONPATH={toxinidir}/lib commands = - pylint: bash -c 'rm -rf build && PYTHONPATH=$PWD/lib:$PWD/repoman/lib pylint bin/* lib/* repoman/bin/* repoman/lib/*' + pylint: bash -c 'rm -rf build && PYTHONPATH=$PWD/lib pylint bin/* lib/*' python -b -Wd setup.py test - bash -c 'if python -c "import lxml.etree"; then python -b -Wd repoman/setup.py test; else echo "repoman tests skipped due to lxml breakage"; fi' |