WIP move project to uv

gerbonara/tests/conftest.py

@@ -1,89 +0,0 @@
-
-import os
-from pathlib import Path
-import tqdm
-import multiprocessing.pool
-import subprocess
-from itertools import chain
-
-import pytest
-
-from .image_support import ImageDifference, run_cargo_cmd, bulk_populate_kicad_fp_export_cache, KICAD_CONTAINER
-
-def pytest_assertrepr_compare(op, left, right):
-    if isinstance(left, ImageDifference) or isinstance(right, ImageDifference):
-        diff = left if isinstance(left, ImageDifference) else right
-        return [
-            f'Image difference assertion failed.',
-            f'    Calculated difference: {diff}',
-            f'    Histogram: {diff.histogram}', ]
-
-
-# store report in node object so tmp_gbr can determine if the test failed.
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    outcome = yield
-    rep = outcome.get_result()
-    setattr(item, f'rep_{rep.when}', rep)
-
-
-fail_dir = Path('gerbonara_test_failures')
-def pytest_sessionstart(session):
-    if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
-        return
-
-    for f in chain(fail_dir.glob('*.gbr'), fail_dir.glob('*.png')):
-        f.unlink()
-
-    try:
-        run_cargo_cmd('resvg', '--help', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except FileNotFoundError:
-        pytest.exit('resvg binary not found, aborting test.', 2)
-
-
-def pytest_configure(config):
-    os.nice(20)
-    # Resvg can sometimes consume a lot of memory. Make sure we don't kill the user's session.
-    if (oom_adj := Path('/proc/self/oom_adj')).is_file():
-        oom_adj.write_text('15\n')
-
-    if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
-        return
-
-    if (lib_dir := os.environ.get('KICAD_FOOTPRINTS')):
-        lib_dir = Path(lib_dir).expanduser()
-        if not lib_dir.is_dir():
-            raise ValueError(f'Path "{lib_dir}" given by KICAD_FOOTPRINTS environment variable does not exist or is not a directory.')
-
-        print('Updating podman image')
-        subprocess.run(['podman', 'pull', KICAD_CONTAINER], check=True)
-
-        print('Checking and bulk re-building KiCad footprint library cache')
-        with multiprocessing.pool.ThreadPool() as pool: # use thread pool here since we're only monitoring podman processes 
-            lib_dirs = list(lib_dir.glob('*.pretty'))
-            res = list(tqdm.tqdm(pool.imap(lambda path: bulk_populate_kicad_fp_export_cache(path), lib_dirs), total=len(lib_dirs)))
-
-
-def pytest_addoption(parser):
-    parser.addoption('--kicad-symbol-library', nargs='*', help='Run symbol library tests on given symbol libraries. May be given multiple times.')
-    parser.addoption('--kicad-footprint-files', nargs='*', help='Run footprint library tests on given footprint files. May be given multiple times.')
-
-
-def pytest_generate_tests(metafunc):
-    if 'kicad_library_file' in metafunc.fixturenames:
-        if not (library_files := metafunc.config.getoption('symbol_library', None)):
-            if (lib_dir := os.environ.get('KICAD_SYMBOLS')):
-                lib_dir = Path(lib_dir).expanduser()
-                library_files = list(lib_dir.glob('*.kicad_sym'))
-            else:
-                raise ValueError('Either --kicad-symbol-library command line parameter or KICAD_SYMBOLS environment variable must be given to run kicad symbol tests.')
-        metafunc.parametrize('kicad_library_file', library_files, ids=list(map(str, library_files)))
-
-    if 'kicad_mod_file' in metafunc.fixturenames:
-        if not (mod_files := metafunc.config.getoption('footprint_files', None)):
-            if (lib_dir := os.environ.get('KICAD_FOOTPRINTS')):
-                lib_dir = Path(lib_dir).expanduser()
-                mod_files = list(lib_dir.glob('*.pretty/*.kicad_mod'))
-            else:
-                raise ValueError('Either --kicad-footprint-files command line parameter or KICAD_FOOTPRINTS environment variable must be given to run kicad footprint tests.')
-        metafunc.parametrize('kicad_mod_file', mod_files, ids=list(map(str, mod_files)))

gerbonara/tests/image_support.py

@@ -1,360 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright 2022 Jan Sebastian Götte <gerbonara@jaseg.de>
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Based on https://github.com/tracespace/tracespace
-#
-
-import subprocess
-from pathlib import Path
-import tempfile
-import textwrap
-import os
-import sys
-import stat
-import random
-import statistics
-from functools import total_ordering
-import shutil
-import bs4
-from contextlib import contextmanager
-import hashlib
-
-import tqdm
-import numpy as np
-from PIL import Image
-
-cachedir = Path(__file__).parent / 'image_cache'
-cachedir.mkdir(exist_ok=True)
-
-KICAD_CONTAINER = os.environ.get('KICAD_CONTAINER', 'registry.hub.docker.com/kicad/kicad:nightly')
-
-@total_ordering
-class ImageDifference:
-    def __init__(self, value, histogram):
-        self.value = value
-        self.histogram = histogram
-
-    def __float__(self):
-        return float(self.value)
-
-    def __eq__(self, other):
-        return float(self) == float(other)
-
-    def __lt__(self, other):
-        return float(self) < float(other)
-
-    def __str__(self):
-        return str(float(self))
-
-@total_ordering
-class Histogram:
-    def __init__(self, value, size):
-        self.value, self.size = value, size
-
-    def __eq__(self, other):
-        other = np.array(other)
-        other[other == None] = self.value[other == None]
-        return (self.value == other).all()
-
-    def __lt__(self, other):
-        other = np.array(other)
-        other[other == None] = self.value[other == None]
-        return (self.value <= other).all()
-
-    def __getitem__(self, index):
-        return self.value[index]
-
-    def __str__(self):
-        return f'{list(self.value)} size={self.size}'
-
-
-def run_cargo_cmd(cmd, args, **kwargs):
-    if cmd.upper() in os.environ:
-        return subprocess.run([os.environ[cmd.upper()], *args], **kwargs)
-
-    try:
-        return subprocess.run([cmd, *args], **kwargs)
-
-    except FileNotFoundError:
-        return subprocess.run([str(Path.home() / '.cargo' / 'bin' / cmd), *args], **kwargs)
-
-def svg_to_png(in_svg, out_png, dpi=100, bg=None):
-    params = f'{dpi}{bg}'.encode()
-    digest = hashlib.blake2b(Path(in_svg).read_bytes() + params).hexdigest()
-    cachefile = cachedir / f'{digest}.png'
-
-    if not cachefile.is_file():
-        bg = 'black' if bg is None else bg
-        run_cargo_cmd('resvg', ['--background', bg, '--dpi', str(dpi), in_svg, cachefile], check=True, stdout=subprocess.DEVNULL)
-
-    shutil.copy(cachefile, out_png)
-
-to_gerbv_svg_units = lambda val, unit='mm': val*72 if unit == 'inch' else val/25.4*72
-
-def gerbv_export(in_gbr, out_svg, export_format='svg', origin=(0, 0), size=(6, 6), fg='#ffffff', bg='#000000', override_unit_spec=None):
-    params = f'{origin}{size}{fg}{bg}'.encode()
-    digest = hashlib.blake2b(Path(in_gbr).read_bytes() + params).hexdigest()
-    cachefile = cachedir / f'{digest}.svg'
-
-    if not cachefile.is_file():
-        print(f'Building cache for {Path(in_gbr).name}')
-        # NOTE: gerbv seems to always export 'clear' polarity apertures as white, irrespective of --foreground, --background
-        # and project file color settings.
-        # TODO: File issue upstream.
-        with tempfile.NamedTemporaryFile('w') as f:
-            if override_unit_spec:
-                units, zeros, digits = override_unit_spec
-                print(f'{Path(in_gbr).name}: overriding excellon unit spec to {units=} {zeros=} {digits=}')
-                units = 0 if units == 'inch' else 1
-                zeros = {None: 0, 'leading': 1, 'trailing': 2}[zeros]
-                unit_spec = textwrap.dedent(f'''(cons 'attribs (list
-                        (list 'autodetect 'Boolean 0)
-                        (list 'zero_suppression 'Enum {zeros})
-                        (list 'units 'Enum {units})
-                        (list 'digits 'Integer {digits})
-                    ))''')
-            else:
-                unit_spec = ''
-
-            r, g, b = int(fg[1:3], 16), int(fg[3:5], 16), int(fg[5:], 16)
-            color = f"(cons 'color #({r*257} {g*257} {b*257}))"
-            f.write(f'''(gerbv-file-version! "2.0A")(define-layer! 0 (cons 'filename "{in_gbr}"){unit_spec}{color})''')
-            f.flush()
-            if override_unit_spec:
-                shutil.copy(f.name, '/tmp/foo.gbv')
-
-            x, y = origin
-            w, h = size
-            cmd = ['gerbv', '-x', export_format,
-                '--border=0',
-                f'--origin={x:.6f}x{y:.6f}', f'--window_inch={w:.6f}x{h:.6f}',
-                f'--background={bg}',
-                f'--foreground={fg}',
-                '-o', str(cachefile), '-p', f.name]
-            subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    else:
-        print(f'Re-using cache for {Path(in_gbr).name}')
-    shutil.copy(cachefile, out_svg)
-
-def kicad_fp_export(mod_file, out_svg):
-    mod_file = Path(mod_file)
-    if mod_file.suffix.lower() != '.kicad_mod':
-        raise ValueError("KiCad footprint file must have .kicad_mod extension for kicad-cli to do it's thing")
-
-    params = f'(noparams)'.encode()
-    digest = hashlib.blake2b(mod_file.read_bytes() + params).hexdigest()
-    cachefile = cachedir / f'{digest}.svg'
-
-    if not cachefile.is_file():
-        print(f'Building cache for {mod_file.name}')
-
-        with tempfile.TemporaryDirectory() as tmpdir:
-            os.chmod(tmpdir, 0o1777)
-            pretty_dir = mod_file.parent
-            fp_name = mod_file.name[:-len('.kicad_mod')]
-            cmd = ['podman', 'run',
-                   '--rm', # Clean up volumes after exit
-                   '--userns=keep-id', # To allow container to read from bind mount
-                   '--mount', f'type=bind,src={pretty_dir},dst=/{pretty_dir.name}',
-                   '--mount', f'type=bind,src={tmpdir},dst=/out',
-                   KICAD_CONTAINER,
-                   'kicad-cli', 'fp', 'export', 'svg', '--output', '/out', '--footprint', fp_name, f'/{pretty_dir.name}']
-            subprocess.run(cmd, check=True) #, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-            out_file = Path(tmpdir) / f'{fp_name}.svg'
-            shutil.copy(out_file, cachefile)
-    else:
-        print(f'Re-using cache for {mod_file.name}')
-    shutil.copy(cachefile, out_svg)
-
-
-def bulk_populate_kicad_fp_export_cache(pretty_dir):
-    def cachefile(mod_file):
-        params = f'(noparams)'.encode()
-        digest = hashlib.blake2b(mod_file.read_bytes() + params).hexdigest()
-        return cachedir / f'{digest}.svg'
-
-    mod_files = list(pretty_dir.glob('*.kicad_mod'))
-    hit_rate = statistics.mean([int(cachefile(fn).is_file())
-                               for fn in random.sample(mod_files, min(len(mod_files), 50))])
-
-    if hit_rate < 0.9:
-        #tqdm.tqdm.write(f'Modfile cache is out of date (hit rate {hit_rate*100:.0f}%), re-building entire cache in bulk')
-
-        with tempfile.TemporaryDirectory() as tmpdir:
-            os.chmod(tmpdir, 0o1777)
-            cmd = ['podman', 'run',
-                   '--rm', # Clean up volumes after exit
-                   '--userns=keep-id', # To allow container to read from bind mount
-                   '--mount', f'type=bind,src={pretty_dir},dst=/{pretty_dir.name}',
-                   '--mount', f'type=bind,src={tmpdir},dst=/out',
-                   KICAD_CONTAINER,
-                   'kicad-cli', 'fp', 'export', 'svg', '--output', '/out', f'/{pretty_dir.name}']
-
-            try:
-                subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL)
-
-            except subprocess.CalledProcessError as e:
-                print('Error running command with command line:', ' '.join(e.cmd), file=sys.stderr)
-                raise e
-
-            for fn in mod_files:
-                out_file = Path(tmpdir) / fn.with_suffix('.svg').name
-                if not out_file.is_file():
-                    tqdm.tqdm.write(f'Output file {out_file} is missing while bulk re-building cache for {pretty_dir}.')
-                else:
-                    shutil.copy(out_file, cachefile(fn))
-
-@contextmanager
-def svg_soup(filename):
-    with open(filename, 'r') as f:
-        soup = bs4.BeautifulSoup(f.read(), 'xml')
-
-    yield soup
-
-    with open(filename, 'w') as f:
-        f.write(str(soup))
-
-def cleanup_gerbv_svg(soup):
-    width = soup.svg["width"]
-    height = soup.svg["height"]
-    width = width[:-2] if width.endswith('pt') else width
-    height = height[:-2] if height.endswith('pt') else height
-    soup.svg['width'] = f'{float(width)/72*25.4:.4f}mm'
-    soup.svg['height'] = f'{float(height)/72*25.4:.4f}mm'
-    for group in soup.find_all('g'):
-        # gerbv uses Cairo's SVG canvas. Cairo's SVG canvas is kind of broken. It has no support for unit
-        # handling at all, which means the output files just end up being in pixels at 72 dpi. Further, it
-        # seems gerbv's aperture macro rendering interacts poorly with Cairo's SVG export. gerbv renders
-        # aperture macros into a new surface, which for some reason gets clipped by Cairo to the given
-        # canvas size. This is just wrong, so we just nuke the clip path from these SVG groups here.
-        #
-        # Apart from being graphically broken, this additionally causes very bad rendering performance.
-        del group['clip-path']
-
-def gerber_difference(reference, actual, diff_out=None, svg_transform=None, size=(10,10), ref_unit_spec=None):
-    with tempfile.NamedTemporaryFile(suffix='.svg') as act_svg,\
-        tempfile.NamedTemporaryFile(suffix='.svg') as ref_svg:
-
-        gerbv_export(reference, ref_svg.name, size=size, export_format='svg', override_unit_spec=ref_unit_spec)
-        gerbv_export(actual, act_svg.name, size=size, export_format='svg')
-
-        with svg_soup(ref_svg.name) as soup:
-            if svg_transform is not None:
-                svg = soup.svg
-                children = list(svg.children)
-                g = soup.new_tag('g', attrs={'transform': svg_transform})
-                for c in children:
-                    g.append(c.extract())
-                svg.append(g)
-
-            cleanup_gerbv_svg(soup)
-
-        with svg_soup(act_svg.name) as soup:
-            cleanup_gerbv_svg(soup)
-
-        return svg_difference(ref_svg.name, act_svg.name, diff_out=diff_out)
-
-def gerber_difference_merge(ref1, ref2, actual, diff_out=None, composite_out=None, svg_transform1=None, svg_transform2=None, size=(10,10)):
-    with tempfile.NamedTemporaryFile(suffix='.svg') as act_svg,\
-        tempfile.NamedTemporaryFile(suffix='.svg') as ref1_svg,\
-        tempfile.NamedTemporaryFile(suffix='.svg') as ref2_svg:
-
-        gerbv_export(ref1, ref1_svg.name, size=size, export_format='svg')
-        gerbv_export(ref2, ref2_svg.name, size=size, export_format='svg')
-        gerbv_export(actual, act_svg.name, size=size, export_format='svg')
-        for var in ['ref1_svg', 'ref2_svg', 'act_svg']:
-            print(f'=== {var} ===')
-            print(Path(locals()[var].name).read_text().splitlines()[1])
-
-        with svg_soup(ref1_svg.name) as soup1:
-            if svg_transform1 is not None:
-                svg = soup1.svg
-                children = list(svg.children)
-                g = soup1.new_tag('g', attrs={'transform': svg_transform1})
-                for c in children:
-                    g.append(c.extract())
-                svg.append(g)
-            cleanup_gerbv_svg(soup1)
-
-            with svg_soup(ref2_svg.name) as soup2:
-                if svg_transform2 is not None:
-                    svg = soup2.svg
-                    children = list(svg.children)
-                    g = soup2.new_tag('g', attrs={'transform': svg_transform2})
-                    for c in children:
-                        g.append(c.extract())
-                    svg.append(g)
-                cleanup_gerbv_svg(soup2)
-
-                defs1 = soup1.find('defs')
-                if not defs1:
-                    defs1 = soup1.new_tag('defs')
-                    soup1.find('svg').insert(0, defs1)
-
-                defs2 = soup2.find('defs')
-                if defs2:
-                    defs2 = defs2.extract()
-                    # explicitly convert .contents into list here and below because else bs4 stumbles over itself
-                    # iterating because we modify the tree in the loop body.
-                    for c in list(defs2.contents):
-                        if hasattr(c, 'attrs'):
-                            c['id'] = 'gn-merge-b-' + c.attrs.get('id', str(id(c)))
-                        defs1.append(c)
-
-                for use in soup2.find_all('use', recursive=True):
-                    if (href := use.get('xlink:href', '')).startswith('#'):
-                        use['xlink:href'] = f'#gn-merge-b-{href[1:]}'
-
-                svg1 = soup1.find('svg')
-                for c in list(soup2.find('svg').contents):
-                    if hasattr(c, 'attrs'):
-                        c['id'] = 'gn-merge-b-' + c.attrs.get('id', str(id(c)))
-                    svg1.append(c)
-
-        if composite_out:
-            shutil.copyfile(ref1_svg.name, composite_out)
-
-        with svg_soup(act_svg.name) as soup:
-            cleanup_gerbv_svg(soup)
-
-        return svg_difference(ref1_svg.name, act_svg.name, diff_out=diff_out)
-
-def svg_difference(reference, actual, diff_out=None, background=None, dpi=100):
-    with tempfile.NamedTemporaryFile(suffix='-ref.png') as ref_png,\
-        tempfile.NamedTemporaryFile(suffix='-act.png') as act_png:
-
-        svg_to_png(reference, ref_png.name, bg=background, dpi=dpi)
-        svg_to_png(actual, act_png.name, bg=background, dpi=dpi)
-
-        return image_difference(ref_png.name, act_png.name, diff_out=diff_out)
-
-def image_difference(reference, actual, diff_out=None):
-    ref = np.array(Image.open(reference)).astype(float)
-    out = np.array(Image.open(actual)).astype(float)
-
-    ref, out = ref.mean(axis=2), out.mean(axis=2) # convert to grayscale
-    # TODO blur images here before comparison to mitigate aliasing issue
-    delta = np.abs(out - ref).astype(float) / 255
-    if diff_out:
-        Image.fromarray((delta*255).astype(np.uint8), mode='L').save(diff_out)
-
-    hist, _bins = np.histogram(delta, bins=10, range=(0, 1))
-    return (ImageDifference(delta.mean(), hist),
-            ImageDifference(delta.max(), hist),
-            Histogram(hist, out.size))
-
-

pyproject.toml

@@ -0,0 +1,69 @@
+[project]
+name = "gerbonara"
+version = "1.5.0"
+description = "Tools to handle Gerber and Excellon files in Python"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.12"
+dependencies = ["click", "rtree", "quart"]
+
+authors = [
+  { name = "jaseg" },
+  { name = "XenGi" },
+]
+
+maintainers = [
+  { name = "Gerbonara maintainers", email = "gerbonara@jaseg.de" },
+]
+
+keywords = ["gerber", "excellon", "pcb", "RS274x", "EDA"]
+
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Information Technology",
+  "Intended Audience :: Manufacturing",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Topic :: Artistic Software",
+  "Topic :: Multimedia :: Graphics",
+  "Topic :: Printing",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)",
+  "Topic :: Scientific/Engineering :: Image Processing",
+  "Topic :: Utilities",
+]
+
+[project.urls]
+Homepage = "https://jaseg.de/projects/gerbonara/"
+Documentation = "https://gerbolyze.gitlab.io/gerbonara/"
+Source = "https://git.jaseg.de/gerbonara.git"
+Tracker = "https://gitlab.com/gerbolyze/gerbonara/issues"
+
+[project.scripts]
+gerbonara = "gerbonara.cli:cli"
+
+[dependency-groups]
+dev = [
+    "pytest",
+    "pytest-xdist",
+    "numpy",
+    "scipy",
+    "tqdm",
+    "beautifulsoup4",
+    "pillow"
+    ]
+
+[build-system]
+requires = ["uv-build"]
+build-backend = "uv_build"
+
+[tool.pytest]
+testpaths = ["tests"]
+norecursedirs = ["*"]
+kicad_symbols_tag = "9.0.6"
+kicad_footprints_tag = "9.0.6"
+# Tag to use for container for footprint svg export
+# For a list of available tags, see https://hub.docker.com/r/kicad/kicad/tags
+kicad_container_tag = "9.0.6-full"

pytest.ini

@@ -1,3 +0,0 @@
-[pytest]
-testpaths = gerbonara/tests
-norecursedirs=*

setup.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-
-from pathlib import Path
-from setuptools import setup, find_packages
-import subprocess
-
-def version():
-    try:
-        res = subprocess.run(['git', 'describe', '--tags', '--match', 'v*'], capture_output=True, check=True, text=True)
-        version, _, _rest = res.stdout.strip()[1:].partition('-')
-        return version
-    except:
-        subprocess.run(['git', 'describe', '--tags', '--match', 'v*'])
-        raise
-
-setup(
-    name='gerbonara',
-    version=version(),
-    author='jaseg, XenGi',
-    author_email='gerbonara@jaseg.de',
-    description='Tools to handle Gerber and Excellon files in Python',
-    long_description=Path('README.md').read_text(),
-    long_description_content_type='text/markdown',
-    url='https://gitlab.com/gerbolyze/gerbonara',
-    project_urls={
-        'Documentation': 'https://gerbolyze.gitlab.io/gerbonara/',
-        # 'Funding': 'https://donate.pypi.org',
-        # 'Say Thanks!': 'http://saythanks.io/to/example',
-        'Source': 'https://gitlab.com/gerbolyze/gerbonara',
-        'Tracker': 'https://gitlab.com/gerbolyze/gerbonara/issues',
-    },
-    packages=find_packages(exclude=['tests']),
-    include_package_data=True,
-    install_requires=['click', 'rtree', 'quart'],
-    entry_points={
-        'console_scripts': [
-            'gerbonara = gerbonara.cli:cli',
-        ],
-    },
-    classifiers=[
-        'Development Status :: 4 - Beta',
-        #'Development Status :: 5 - Production/Stable',
-        'Environment :: Console',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Information Technology',
-        'Intended Audience :: Manufacturing',
-        'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: Apache Software License',
-        'Natural Language :: English',
-        'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Topic :: Artistic Software',
-        'Topic :: Multimedia :: Graphics',
-        'Topic :: Printing',
-        'Topic :: Scientific/Engineering',
-        'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)',
-        'Topic :: Scientific/Engineering :: Image Processing',
-        'Topic :: Utilities',
-        'Typing :: Typed',
-    ],
-    keywords='gerber excellon pcb',
-    python_requires='>=3.10',
-)

src/gerbonara/__init__.py

@@ -30,5 +30,6 @@ from .excellon import ExcellonFile
 from .ipc356 import Netlist
 from .layers import LayerStack
 from .utils import MM, Inch
+from importlib.metadata import version
 
-__version__ = '1.5.0'
+__version__ = version('gerbonara')

tests/conftest.py

@@ -0,0 +1,129 @@
+
+import os
+from pathlib import Path
+import tqdm
+import multiprocessing.pool
+import subprocess
+from itertools import chain
+
+import pytest
+
+from .image_support import ImageDifference, run_cargo_cmd, ImageSupport
+
+
+@pytest.fixture
+def kicad_container(request):
+    return request.config.kicad_container
+
+
+@pytest.fixture()
+def kicad_footprints_libdir(request):
+    return request.config.kicad_footprints_libdir
+
+
+@pytest.fixture()
+def kicad_symbols_libdir(request):
+    return request.config.kicad_symbols_libdir
+
+
+@pytest.fixture()
+def img_support(request):
+    return request.config.image_support
+
+
+def pytest_assertrepr_compare(op, left, right):
+    if isinstance(left, ImageDifference) or isinstance(right, ImageDifference):
+        diff = left if isinstance(left, ImageDifference) else right
+        return [
+            f'Image difference assertion failed.',
+            f'    Calculated difference: {diff}',
+            f'    Histogram: {diff.histogram}', ]
+
+
+# store report in node object so tmp_gbr can determine if the test failed.
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    outcome = yield
+    rep = outcome.get_result()
+    setattr(item, f'rep_{rep.when}', rep)
+
+
+fail_dir = Path('gerbonara_test_failures')
+def pytest_sessionstart(session):
+    if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
+        return
+
+    for f in chain(fail_dir.glob('*.gbr'), fail_dir.glob('*.png')):
+        f.unlink()
+
+    try:
+        run_cargo_cmd('resvg', '--help', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    except FileNotFoundError:
+        pytest.exit('resvg binary not found, aborting test.', 2)
+
+
+def _update_repo_cache(lib_dir, repo_url, tag):
+    if not lib_dir.is_dir():
+        print(f'Checking out KiCad footprint repo tag {tag}')
+        subprocess.run(['git', '-c', 'advice.detachedHead=false', 'clone', '--branch', tag, '--depth', '1', repo_url, str(lib_dir)], check=True)
+    else:
+        print(f'Found cached KiCad footprint checkout, updating to {tag}')
+        subprocess.run(['git', '-C', str(lib_dir), 'fetch', '--depth', '1', 'origin', tag], check=True)
+        subprocess.run(['git', '-c', 'advice.detachedHead=false', '-C', str(lib_dir), 'reset', '--hard', tag], check=True)
+        subprocess.run(['git', '-C', str(lib_dir), 'clean', '--force', '-d', '-x'], check=True)
+
+
+def pytest_addoption(parser):
+    parser.addini('kicad_footprints_tag', 'git tag or branch for KiCad footprint library repo used as testdata', default='main')
+    parser.addini('kicad_symbols_tag', 'git tag or branch for KiCad symbol library repo used as testdata', default='main')
+    parser.addini('kicad_container_tag', 'docker hub tag for the KiCad container to use for exporting footprint images', default='main')
+
+
+def pytest_configure(config):
+    os.nice(20)
+    # Resvg can sometimes consume a lot of memory. Make sure we don't kill the user's session.
+    if (oom_adj := Path('/proc/self/oom_adj')).is_file():
+        oom_adj.write_text('15\n')
+
+    if 'PYTEST_XDIST_WORKER' in os.environ: # only run this on the controller
+        return
+
+    if (lib_dir := os.environ.get('KICAD_FOOTPRINTS')):
+        config.kicad_footprints_libdir = Path(lib_dir).expanduser()
+    else:
+        config.kicad_footprints_libdir = config.cache.mkdir('kicad-footprints') / 'repo'
+
+    if (lib_dir := os.environ.get('KICAD_SYMBOLS')):
+        config.kicad_symbols_libdir = Path(lib_dir).expanduser()
+    else:
+        config.kicad_symbols_libdir = config.cache.mkdir('kicad-symbols') / 'repo'
+
+    # Update cached library repos unless they are overridden from outside.
+    if not os.environ.get('KICAD_FOOTPRINTS'):
+        tag = config.getini('kicad_footprints_tag')
+        _update_repo_cache(config.kicad_footprints_libdir, 'https://gitlab.com/kicad/libraries/kicad-footprints', tag)
+
+    if not os.environ.get('KICAD_SYMBOLS'):
+        tag = config.getini('kicad_symbols_tag')
+        _update_repo_cache(config.kicad_symbols_libdir, 'https://gitlab.com/kicad/libraries/kicad-symbols', tag)
+
+    print('Updating podman image')
+    tag = config.getini("kicad_container_tag")
+    config.kicad_container = os.environ.get('KICAD_CONTAINER', f'registry.hub.docker.com/kicad/kicad:{tag}')
+    subprocess.run(['podman', 'pull', config.kicad_container], check=True)
+
+    config.image_support = ImageSupport(config.cache.mkdir('image_cache'), config.kicad_container)
+    print('Checking KiCad footprint library render cache')
+    with multiprocessing.pool.ThreadPool() as pool: # use thread pool here since we're only monitoring podman processes 
+        lib_dirs = list(config.kicad_footprints_libdir.glob('*.pretty'))
+        res = list(tqdm.tqdm(pool.imap(lambda path: config.image_support.bulk_populate_kicad_fp_export_cache(path), lib_dirs), total=len(lib_dirs)))
+
+
+def pytest_generate_tests(metafunc):
+    if 'kicad_library_file' in metafunc.fixturenames:
+        library_files = list(kicad_symbols_libdir(metafunc.config).glob('*.kicad_sym'))
+        metafunc.parametrize('kicad_library_file', library_files, ids=list(map(str, library_files)))
+
+    if 'kicad_mod_file' in metafunc.fixturenames:
+        mod_files = list(kicad_footprints_libdir(metafunc.config).glob('*.pretty/*.kicad_mod'))
+        metafunc.parametrize('kicad_mod_file', mod_files, ids=list(map(str, mod_files)))

tests/image_support.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright 2022 Jan Sebastian Götte <gerbonara@jaseg.de>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Based on https://github.com/tracespace/tracespace
+#
+
+import subprocess
+from pathlib import Path
+import tempfile
+import textwrap
+import os
+import sys
+import stat
+import random
+import statistics
+from functools import total_ordering
+import shutil
+import bs4
+from contextlib import contextmanager
+import hashlib
+
+import tqdm
+import numpy as np
+from PIL import Image
+
+@total_ordering
+class ImageDifference:
+    def __init__(self, value, histogram):
+        self.value = value
+        self.histogram = histogram
+
+    def __float__(self):
+        return float(self.value)
+
+    def __eq__(self, other):
+        return float(self) == float(other)
+
+    def __lt__(self, other):
+        return float(self) < float(other)
+
+    def __str__(self):
+        return str(float(self))
+
+@total_ordering
+class Histogram:
+    def __init__(self, value, size):
+        self.value, self.size = value, size
+
+    def __eq__(self, other):
+        other = np.array(other)
+        other[other == None] = self.value[other == None]
+        return (self.value == other).all()
+
+    def __lt__(self, other):
+        other = np.array(other)
+        other[other == None] = self.value[other == None]
+        return (self.value <= other).all()
+
+    def __getitem__(self, index):
+        return self.value[index]
+
+    def __str__(self):
+        return f'{list(self.value)} size={self.size}'
+
+
+def run_cargo_cmd(cmd, args, **kwargs):
+    if cmd.upper() in os.environ:
+        return subprocess.run([os.environ[cmd.upper()], *args], **kwargs)
+
+    try:
+        return subprocess.run([cmd, *args], **kwargs)
+
+    except FileNotFoundError:
+        return subprocess.run([str(Path.home() / '.cargo' / 'bin' / cmd), *args], **kwargs)
+
+
+to_gerbv_svg_units = lambda val, unit='mm': val*72 if unit == 'inch' else val/25.4*72
+
+
+@contextmanager
+def svg_soup(filename):
+    with open(filename, 'r') as f:
+        soup = bs4.BeautifulSoup(f.read(), 'xml')
+
+    yield soup
+
+    with open(filename, 'w') as f:
+        f.write(str(soup))
+
+
+class ImageSupport:
+    def __init__(self, cachedir, kicad_container):
+        self.cache_root = cachedir
+        self.kicad_container = kicad_container
+
+
+    def cachedir(self, scope, filename, suffix):
+        return self.cache_root / scope / f'{filename}.{suffix}'
+
+
+    def svg_to_png(self, in_svg, out_png, dpi=100, bg=None):
+        params = f'{dpi}{bg}'.encode()
+        digest = hashlib.blake2b(Path(in_svg).read_bytes() + params).hexdigest()
+        cachefile = self.cachedir('svg_render', digest, 'png')
+
+        if not cachefile.is_file():
+            bg = 'black' if bg is None else bg
+            run_cargo_cmd('resvg', ['--background', bg, '--dpi', str(dpi), in_svg, cachefile], check=True, stdout=subprocess.DEVNULL)
+
+        shutil.copy(cachefile, out_png)
+
+
+    def gerbv_export(self, in_gbr, out_svg, export_format='svg', origin=(0, 0), size=(6, 6), fg='#ffffff', bg='#000000', override_unit_spec=None):
+        params = f'{origin}{size}{fg}{bg}'.encode()
+        digest = hashlib.blake2b(Path(in_gbr).read_bytes() + params).hexdigest()
+        cachefile = self.cachedir('gerbv-export', digest, 'svg')
+
+        if not cachefile.is_file():
+            print(f'Building cache for {Path(in_gbr).name}')
+            # NOTE: gerbv seems to always export 'clear' polarity apertures as white, irrespective of --foreground, --background
+            # and project file color settings.
+            # TODO: File issue upstream.
+            with tempfile.NamedTemporaryFile('w') as f:
+                if override_unit_spec:
+                    units, zeros, digits = override_unit_spec
+                    print(f'{Path(in_gbr).name}: overriding excellon unit spec to {units=} {zeros=} {digits=}')
+                    units = 0 if units == 'inch' else 1
+                    zeros = {None: 0, 'leading': 1, 'trailing': 2}[zeros]
+                    unit_spec = textwrap.dedent(f'''(cons 'attribs (list
+                            (list 'autodetect 'Boolean 0)
+                            (list 'zero_suppression 'Enum {zeros})
+                            (list 'units 'Enum {units})
+                            (list 'digits 'Integer {digits})
+                        ))''')
+                else:
+                    unit_spec = ''
+
+                r, g, b = int(fg[1:3], 16), int(fg[3:5], 16), int(fg[5:], 16)
+                color = f"(cons 'color #({r*257} {g*257} {b*257}))"
+                f.write(f'''(gerbv-file-version! "2.0A")(define-layer! 0 (cons 'filename "{in_gbr}"){unit_spec}{color})''')
+                f.flush()
+                if override_unit_spec:
+                    shutil.copy(f.name, '/tmp/foo.gbv')
+
+                x, y = origin
+                w, h = size
+                cmd = ['gerbv', '-x', export_format,
+                    '--border=0',
+                    f'--origin={x:.6f}x{y:.6f}', f'--window_inch={w:.6f}x{h:.6f}',
+                    f'--background={bg}',
+                    f'--foreground={fg}',
+                    '-o', str(cachefile), '-p', f.name]
+                subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        else:
+            print(f'Re-using cache for {Path(in_gbr).name}')
+        shutil.copy(cachefile, out_svg)
+
+
+    def kicad_fp_export(self, mod_file, out_svg):
+        mod_file = Path(mod_file)
+        if mod_file.suffix.lower() != '.kicad_mod':
+            raise ValueError("KiCad footprint file must have .kicad_mod extension for kicad-cli to do it's thing")
+
+        params = f'(noparams)'.encode()
+        digest = hashlib.blake2b(mod_file.read_bytes() + params).hexdigest()
+        cachefile = self.cachedir('kicad-footprint-export', digest, 'svg')
+
+        if not cachefile.is_file():
+            print(f'Building cache for {mod_file.name}')
+
+            with tempfile.TemporaryDirectory() as tmpdir:
+                os.chmod(tmpdir, 0o1777)
+                pretty_dir = mod_file.parent
+                fp_name = mod_file.name[:-len('.kicad_mod')]
+                cmd = ['podman', 'run',
+                       '--rm', # Clean up volumes after exit
+                       '--userns=keep-id', # To allow container to read from bind mount
+                       '--mount', f'type=bind,src={pretty_dir},dst=/{pretty_dir.name}',
+                       '--mount', f'type=bind,src={tmpdir},dst=/out',
+                       self.kicad_container,
+                       'kicad-cli', 'fp', 'export', 'svg', '--output', '/out', '--footprint', fp_name, f'/{pretty_dir.name}']
+                subprocess.run(cmd, check=True) #, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                out_file = Path(tmpdir) / f'{fp_name}.svg'
+                shutil.copy(out_file, cachefile)
+        else:
+            print(f'Re-using cache for {mod_file.name}')
+        shutil.copy(cachefile, out_svg)
+
+
+    def bulk_populate_kicad_fp_export_cache(self, pretty_dir):
+        def cachefile(mod_file):
+            params = f'(noparams)'.encode()
+            digest = hashlib.blake2b(mod_file.read_bytes() + params).hexdigest()
+            return self.cachedir('kicad-footprint-export', digest, 'svg')
+
+        mod_files = list(pretty_dir.glob('*.kicad_mod'))
+        hit_rate = statistics.mean([int(cachefile(fn).is_file())
+                                   for fn in random.sample(mod_files, min(len(mod_files), 50))])
+
+        if hit_rate < 0.9:
+            #tqdm.tqdm.write(f'Modfile cache is out of date (hit rate {hit_rate*100:.0f}%), re-building entire cache in bulk')
+
+            with tempfile.TemporaryDirectory() as tmpdir:
+                os.chmod(tmpdir, 0o1777)
+                cmd = ['podman', 'run',
+                       '--rm', # Clean up volumes after exit
+                       '--userns=keep-id', # To allow container to read from bind mount
+                       '--mount', f'type=bind,src={pretty_dir},dst=/{pretty_dir.name}',
+                       '--mount', f'type=bind,src={tmpdir},dst=/out',
+                       self.kicad_container,
+                       'kicad-cli', 'fp', 'export', 'svg', '--output', '/out', f'/{pretty_dir.name}']
+                print(f'Running "{" ".join(cmd)}"')
+
+                try:
+                    subprocess.run(cmd, check=True) #, stdout=subprocess.DEVNULL)
+
+                except subprocess.CalledProcessError as e:
+                    print('Error running command with command line:', ' '.join(e.cmd), file=sys.stderr)
+                    raise e
+
+                for fn in mod_files:
+                    out_file = Path(tmpdir) / fn.with_suffix('.svg').name
+                    if not out_file.is_file():
+                        tqdm.tqdm.write(f'Output file {out_file} is missing while bulk re-building cache for {pretty_dir}.')
+                    else:
+                        shutil.copy(out_file, cachefile(fn))
+
+
+    def cleanup_gerbv_svg(self, soup):
+        width = soup.svg["width"]
+        height = soup.svg["height"]
+        width = width[:-2] if width.endswith('pt') else width
+        height = height[:-2] if height.endswith('pt') else height
+        soup.svg['width'] = f'{float(width)/72*25.4:.4f}mm'
+        soup.svg['height'] = f'{float(height)/72*25.4:.4f}mm'
+        for group in soup.find_all('g'):
+            # gerbv uses Cairo's SVG canvas. Cairo's SVG canvas is kind of broken. It has no support for unit
+            # handling at all, which means the output files just end up being in pixels at 72 dpi. Further, it
+            # seems gerbv's aperture macro rendering interacts poorly with Cairo's SVG export. gerbv renders
+            # aperture macros into a new surface, which for some reason gets clipped by Cairo to the given
+            # canvas size. This is just wrong, so we just nuke the clip path from these SVG groups here.
+            #
+            # Apart from being graphically broken, this additionally causes very bad rendering performance.
+            del group['clip-path']
+
+    def gerber_difference(self, reference, actual, diff_out=None, svg_transform=None, size=(10,10), ref_unit_spec=None):
+        with tempfile.NamedTemporaryFile(suffix='.svg') as act_svg,\
+            tempfile.NamedTemporaryFile(suffix='.svg') as ref_svg:
+
+            self.gerbv_export(reference, ref_svg.name, size=size, export_format='svg', override_unit_spec=ref_unit_spec)
+            self.gerbv_export(actual, act_svg.name, size=size, export_format='svg')
+
+            with svg_soup(ref_svg.name) as soup:
+                if svg_transform is not None:
+                    svg = soup.svg
+                    children = list(svg.children)
+                    g = soup.new_tag('g', attrs={'transform': svg_transform})
+                    for c in children:
+                        g.append(c.extract())
+                    svg.append(g)
+
+                self.cleanup_gerbv_svg(soup)
+
+            with svg_soup(act_svg.name) as soup:
+                self.cleanup_gerbv_svg(soup)
+
+            return self.svg_difference(ref_svg.name, act_svg.name, diff_out=diff_out)
+
+    def gerber_difference_merge(self, ref1, ref2, actual, diff_out=None, composite_out=None, svg_transform1=None, svg_transform2=None, size=(10,10)):
+        with tempfile.NamedTemporaryFile(suffix='.svg') as act_svg,\
+            tempfile.NamedTemporaryFile(suffix='.svg') as ref1_svg,\
+            tempfile.NamedTemporaryFile(suffix='.svg') as ref2_svg:
+
+            self.gerbv_export(ref1, ref1_svg.name, size=size, export_format='svg')
+            self.gerbv_export(ref2, ref2_svg.name, size=size, export_format='svg')
+            self.gerbv_export(actual, act_svg.name, size=size, export_format='svg')
+            for var in ['ref1_svg', 'ref2_svg', 'act_svg']:
+                print(f'=== {var} ===')
+                print(Path(locals()[var].name).read_text().splitlines()[1])
+
+            with svg_soup(ref1_svg.name) as soup1:
+                if svg_transform1 is not None:
+                    svg = soup1.svg
+                    children = list(svg.children)
+                    g = soup1.new_tag('g', attrs={'transform': svg_transform1})
+                    for c in children:
+                        g.append(c.extract())
+                    svg.append(g)
+                self.cleanup_gerbv_svg(soup1)
+
+                with svg_soup(ref2_svg.name) as soup2:
+                    if svg_transform2 is not None:
+                        svg = soup2.svg
+                        children = list(svg.children)
+                        g = soup2.new_tag('g', attrs={'transform': svg_transform2})
+                        for c in children:
+                            g.append(c.extract())
+                        svg.append(g)
+                    self.cleanup_gerbv_svg(soup2)
+
+                    defs1 = soup1.find('defs')
+                    if not defs1:
+                        defs1 = soup1.new_tag('defs')
+                        soup1.find('svg').insert(0, defs1)
+
+                    defs2 = soup2.find('defs')
+                    if defs2:
+                        defs2 = defs2.extract()
+                        # explicitly convert .contents into list here and below because else bs4 stumbles over itself
+                        # iterating because we modify the tree in the loop body.
+                        for c in list(defs2.contents):
+                            if hasattr(c, 'attrs'):
+                                c['id'] = 'gn-merge-b-' + c.attrs.get('id', str(id(c)))
+                            defs1.append(c)
+
+                    for use in soup2.find_all('use', recursive=True):
+                        if (href := use.get('xlink:href', '')).startswith('#'):
+                            use['xlink:href'] = f'#gn-merge-b-{href[1:]}'
+
+                    svg1 = soup1.find('svg')
+                    for c in list(soup2.find('svg').contents):
+                        if hasattr(c, 'attrs'):
+                            c['id'] = 'gn-merge-b-' + c.attrs.get('id', str(id(c)))
+                        svg1.append(c)
+
+            if composite_out:
+                shutil.copyfile(ref1_svg.name, composite_out)
+
+            with svg_soup(act_svg.name) as soup:
+                self.cleanup_gerbv_svg(soup)
+
+            return self.svg_difference(ref1_svg.name, act_svg.name, diff_out=diff_out)
+
+    def svg_difference(self, reference, actual, diff_out=None, background=None, dpi=100):
+        with tempfile.NamedTemporaryFile(suffix='-ref.png') as ref_png,\
+            tempfile.NamedTemporaryFile(suffix='-act.png') as act_png:
+
+            svg_to_png(reference, ref_png.name, bg=background, dpi=dpi)
+            svg_to_png(actual, act_png.name, bg=background, dpi=dpi)
+
+            return self.image_difference(ref_png.name, act_png.name, diff_out=diff_out)
+
+    def image_difference(self, reference, actual, diff_out=None):
+        ref = np.array(Image.open(reference)).astype(float)
+        out = np.array(Image.open(actual)).astype(float)
+
+        ref, out = ref.mean(axis=2), out.mean(axis=2) # convert to grayscale
+        # TODO blur images here before comparison to mitigate aliasing issue
+        delta = np.abs(out - ref).astype(float) / 255
+        if diff_out:
+            Image.fromarray((delta*255).astype(np.uint8), mode='L').save(diff_out)
+
+        hist, _bins = np.histogram(delta, bins=10, range=(0, 1))
+        return (ImageDifference(delta.mean(), hist),
+                ImageDifference(delta.max(), hist),
+                Histogram(hist, out.size))
+
+