Submodule-cache WIP

submodule-cache.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import subprocess
+from os import path
+import os
+import warnings
+import tempfile
+
+def list_submodules():
+    urls = {}
+    for cfg_id, url in list_submodule_prop('url'):
+        urls[cfg_id] = url.decode()
+
+    out = {}
+    for cfg_id, path in list_submodule_prop('path'):
+        if cfg_id not in urls:
+            warnings.warn('Submodule {cfg_id} has no URL configured! Skipping!')
+        else:
+            out[cfg_id] = (urls[cfg_id], path)
+
+    leftover = urls.keys() - out.keys()
+    for mod in leftover:
+        warnings.warn('Submodule {mod} have no path configured! Skipping!')
+
+    for cfg_id, _nocache in list_submodule_prop('noref'):
+        del out[cfg_id]
+
+    return out
+
+def list_submodule_prop(prop):
+    proc = subprocess.run('git config --file .gitmodules --get-regexp'.split() + [f'\.{prop}$'], check=True, capture_output=True)
+    for line in proc.stdout.splitlines():
+        key, value = line.split()
+        #example key: submodule.fw/hid-dials/upstream/st-hal-f0.url
+        _, cfg_name, _ = key.split(b'.')
+
+        cfg_id = cfg_name.decode('utf-8')
+        yield cfg_id, value
+
+def get_submodule_prop(cfg_id, prop):
+    try:
+        proc = subprocess.run('git config --file .gitmodules --get'.split() + [f'submodule.{cfg_id}.{prop}'], check=True, capture_output=True)
+        return proc.stdout.decode().strip()
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            return None # key does not exist
+        raise e
+
+def cache_home():
+    return os.environ.get('XDG_CACHE_HOME',
+            path.join(os.environ['HOME'], '.cache'))
+
+def get_global_cachedir(create=False, verbose=False):
+    le_path = path.join(cache_home(), 'submodule-references')
+    if create:
+        if verbose:
+            print(f'Cache dir {le_path} does not exist. Creating.')
+        os.makedirs(le_path, exist_ok=True)
+    return le_path
+
+def splitdir_run(repo, spec):
+    source_branch, target_branch, prefix = spec.split(':')
+    with tempfile.TemporaryDirectory(prefix='sm-split-') as tmpdir:
+        try:
+            # --force to prevent errors for existing checkouts
+            subprocess.run('git worktree add --force'.split() + [tmpdir, source_branch], check=True, cwd=repo)
+            subprocess.run('git subtree split'.split() + ['-b', target_branch, '--prefix', prefix], check=True, cwd=tmpdir)
+        finally:
+            subprocess.run('git worktree remove'.split() + [tmpdir], check=True, cwd=repo)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--force', action='store_true')
+    parser.add_argument('-q', '--quiet', action='store_true')
+    args = parser.parse_args()
+
+    cachedir = get_global_cachedir(create=True)
+
+    quiet_opts = ['--quiet'] if args.quiet else []
+    force_opts = ['--force'] if args.force else []
+
+    for cfg_id, (url, le_path) in list_submodules().items():
+        short_id = cfg_id.split('/')[-1]
+        refdir = path.join(cachedir, short_id)
+        branch = get_submodule_prop(cfg_id, 'branch')
+        splitdir = get_submodule_prop(cfg_id, 'splitdir')
+        if not path.isdir(refdir):
+            print(f'Submodule {short_id} is not cached. Cloning from {url}...')
+            subprocess.run('git clone --bare'.split() + quiet_opts + [url, refdir], check=True)
+
+        else:
+            print(f'Updating submodule {short_id}...')
+
+            #Sanity check
+            proc = subprocess.run('git remote get-url origin'.split(), cwd=refdir, check=True, capture_output=True)
+            cache_url = proc.stdout.decode().strip()
+            if cache_url != url:
+                raise SystemError(f'This repo configures submodule {short_id} with upstream URL {url}, but the cache uses upstream URL {cache_url}')
+
+            subprocess.run('git fetch'.split() + force_opts, cwd=refdir, check=True)
+
+        if splitdir:
+            splitdir_run(refdir, splitdir)
+
+        print(f'Checking out submodule {short_id}...')
+        subprocess.run('git submodule update --init --reference'.split() + [refdir, le_path])
+