Add support to ignore refs for grok-manifest

With gerrit-3.x, the default behaviour is to hide refs/meta/* from
anonymous access, so when we calculate fingerprints directly in the
repository, they will be different on the replicas due to missing an
entire refname.

It's now possible to pass -r refs/meta/* (plus any number of others) to
create a "view" that matches what anonymous clients get.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

2 files changed, 30 insertions, 14 deletions

diff --git a/grokmirror/__init__.py b/grokmirror/__init__.py
index 980f8a7..4dd191c 100644
--- a/grokmirror/__init__.py
+++ b/grokmirror/__init__.py
@@ -226,7 +226,7 @@ def get_repo_obj_info(fullpath):
     return obj_info
 
 
-def get_repo_defs(toplevel, gitdir, usenow=False):
+def get_repo_defs(toplevel, gitdir, usenow=False, ignorerefs=None):
     fullpath = os.path.join(toplevel, gitdir.lstrip('/'))
     description = None
     try:
@@ -276,7 +276,7 @@ def get_repo_defs(toplevel, gitdir, usenow=False):
     # "state fingerprint" -- basically the output of "git show-ref | sha1sum".
     # git show-ref output is deterministic and should accurately list all refs
     # and their relation to heads/tags/etc.
-    fingerprint = get_repo_fingerprint(toplevel, gitdir, force=True)
+    fingerprint = get_repo_fingerprint(toplevel, gitdir, force=True, ignorerefs=ignorerefs)
     # Record it in the repo for other use
     set_repo_fingerprint(toplevel, gitdir, fingerprint)
     repoinfo = {
@@ -661,7 +661,7 @@ def get_forkgroups(obstdir, toplevel):
     return forkgroups
 
 
-def get_repo_fingerprint(toplevel, gitdir, force=False):
+def get_repo_fingerprint(toplevel, gitdir, force=False, ignorerefs=None):
     fullpath = os.path.join(toplevel, gitdir.lstrip('/'))
     if not os.path.exists(fullpath):
         logger.debug('Cannot fingerprint %s, as it does not exist', fullpath)
@@ -679,9 +679,24 @@ def get_repo_fingerprint(toplevel, gitdir, force=False):
             logger.debug('No heads in %s, nothing to fingerprint.', fullpath)
             return None
 
-        # We add the final "\n" to be compatible with cmdline output
-        # of git-show-ref
-        fingerprint = hashlib.sha1(out.encode() + b"\n").hexdigest()
+        if ignorerefs:
+            hasher = hashlib.sha1()
+            for line in out.split('\n'):
+                rhash, rname = line.split(maxsplit=1)
+                ignored = False
+                for ignoreref in ignorerefs:
+                    if fnmatch.fnmatch(rname, ignoreref):
+                        ignored = True
+                        break
+                if ignored:
+                    continue
+                hasher.update(line.encode() + b'\n')
+
+            fingerprint = hasher.hexdigest()
+        else:
+            # We add the final "\n" to be compatible with cmdline output
+            # of git-show-ref
+            fingerprint = hashlib.sha1(out.encode() + b'\n').hexdigest()
 
         # Save it for future use
         if not force:
diff --git a/grokmirror/manifest.py b/grokmirror/manifest.py
index bbde601..7eb8978 100755
--- a/grokmirror/manifest.py
+++ b/grokmirror/manifest.py
@@ -24,7 +24,7 @@ import grokmirror
 logger = logging.getLogger(__name__)
 
 
-def update_manifest(manifest, toplevel, fullpath, usenow):
+def update_manifest(manifest, toplevel, fullpath, usenow, ignorerefs):
     logger.debug('Examining %s', fullpath)
     if not grokmirror.is_bare_git_repo(fullpath):
         logger.critical('Error opening %s.', fullpath)
@@ -32,14 +32,12 @@ def update_manifest(manifest, toplevel, fullpath, usenow):
         sys.exit(1)
 
     gitdir = '/' + os.path.relpath(fullpath, toplevel)
+    repoinfo = grokmirror.get_repo_defs(toplevel, gitdir, usenow=usenow, ignorerefs=ignorerefs)
     # Ignore it if it's an empty git repository
-    fp = grokmirror.get_repo_fingerprint(toplevel, gitdir, force=True)
-    if not fp:
+    if not repoinfo['fingerprint']:
         logger.info(' manifest: ignored %s (no heads)', gitdir)
         return
 
-    repoinfo = grokmirror.get_repo_defs(toplevel, gitdir, usenow=usenow)
-
     if gitdir not in manifest:
         # In grokmirror-1.x we didn't normalize paths to be always with a leading '/', so
         # check the manifest for both and make sure we only save the path with a leading /
@@ -143,6 +141,8 @@ def parse_args():
     op.add_argument('-i', '--ignore-paths', dest='ignore', action='append',
                     default=None,
                     help='When finding git dirs, ignore these paths (accepts shell-style globbing)')
+    op.add_argument('-r', '--ignore-refs', dest='ignore_refs', action='append', default=None,
+                    help='Refs to exclude from fingerprint calculation (e.g. refs/meta/*)')
     op.add_argument('-w', '--wait-for-manifest', dest='wait',
                     action='store_true', default=False,
                     help='When running with arguments, wait if manifest is not there '
@@ -192,7 +192,8 @@ def parse_args():
 
 def grok_manifest(manifile, toplevel, paths=None, logfile=None, usenow=False,
                   check_export_ok=False, purge=False, remove=False,
-                  pretty=False, ignore=None, wait=False, verbose=False, fetchobst=False):
+                  pretty=False, ignore=None, wait=False, verbose=False, fetchobst=False,
+                  ignorerefs=None):
     global logger
     loglevel = logging.INFO
     logger = grokmirror.init_logger('manifest', logfile, loglevel, verbose)
@@ -264,7 +265,7 @@ def grok_manifest(manifile, toplevel, paths=None, logfile=None, usenow=False,
         if os.path.islink(gitdir):
             symlinks.append(gitdir)
         else:
-            update_manifest(manifest, toplevel, gitdir, usenow)
+            update_manifest(manifest, toplevel, gitdir, usenow, ignorerefs)
             if fetchobst:
                 # Do it after we're done with manifest, to avoid keeping it locked
                 tofetch.add(gitdir)
@@ -300,7 +301,7 @@ def command():
         usenow=opts.usenow, check_export_ok=opts.check_export_ok,
         purge=opts.purge, remove=opts.remove, pretty=opts.pretty,
         ignore=opts.ignore, wait=opts.wait, verbose=opts.verbose,
-        fetchobst=opts.fetchobst)
+        fetchobst=opts.fetchobst, ignorerefs=opts.ignore_refs)
 
 
 if __name__ == '__main__':