Merge pull request #824 from davidgiven/ab

Update ab to the new ninja version.

.github/workflows/ccpp.yml

@@ -88,7 +88,7 @@ jobs:
         wsl --import fedora fedora install.tar.gz
         wsl --set-default fedora
         wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
-        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico'
+        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico ninja-build'
 
     - name: fix line endings
       run: |

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
         wsl --import fedora fedora install.tar.gz
         wsl --set-default fedora
         wsl sh -c 'dnf -y install https://github.com/rpmsphere/noarch/raw/master/r/rpmsphere-release-40-1.noarch.rpm'
-        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico'
+        wsl sh -c 'dnf -y install gcc gcc-c++ protobuf-c-compiler protobuf-devel fmt-devel systemd-devel sqlite-devel wxGTK-devel mingw32-gcc mingw32-gcc-c++ mingw32-zlib-static mingw32-protobuf-static mingw32-sqlite-static mingw32-wxWidgets3-static mingw32-libpng-static mingw32-libjpeg-static mingw32-libtiff-static mingw32-nsis png2ico ninja-build'
 
     - name: fix line endings
       run: |

Makefile

@@ -12,12 +12,15 @@ ifeq ($(BUILDTYPE),windows)
 	MINGW = i686-w64-mingw32-
 	CC = $(MINGW)gcc
 	CXX = $(MINGW)g++ -std=c++20
-	CFLAGS += -g -O3
+	CFLAGS += -g -O3 \
+		-Wno-unknown-warning-option \
+		-ffunction-sections \
+		-fdata-sections
 	CXXFLAGS += \
 		-fext-numeric-literals \
 		-Wno-deprecated-enum-float-conversion \
 		-Wno-deprecated-enum-enum-conversion
-	LDFLAGS += -static
+	LDFLAGS += -static -Wl,--gc-sections
 	AR = $(MINGW)ar
 	PKG_CONFIG = $(MINGW)pkg-config -static
 	WINDRES = $(MINGW)windres
@@ -27,6 +30,8 @@ else
 	CC = gcc
 	CXX = g++ -std=c++20
 	CFLAGS = -g -O3 \
+		-Wno-unknown-warning-option
+	CXXFLAGS += \
 		-Wno-deprecated-enum-float-conversion \
 		-Wno-deprecated-enum-enum-conversion
 	LDFLAGS =

build.py

@@ -8,7 +8,7 @@ import config
 import re
 
 # Hack for building on Fedora/WSL; executables get the .exe extension,
-# build the build system detects it as Linux.
+# but the build system detects it as Linux.
 import build.toolchain
 
 toolchain.Toolchain.EXE = "$(EXT)"
@@ -93,7 +93,7 @@ else:
                     + c[1]
                     + "' '"
                     + c[2]
-                    + "' $(dir $[outs[0]]) > /dev/null"
+                    + "' $[dirname(filenameof(outs[0]))] > /dev/null"
                 ],
                 label="CORPUSTEST",
             )

build/ab.mk

@@ -15,16 +15,17 @@ HOSTCC ?= gcc
 HOSTCXX ?= g++
 HOSTAR ?= ar
 HOSTCFLAGS ?= -g -Og
+HOSTCXXFLAGS ?= $(HOSTCFLAGS)
 HOSTLDFLAGS ?= -g
 
 CC ?= $(HOSTCC)
 CXX ?= $(HOSTCXX)
 AR ?= $(HOSTAR)
 CFLAGS ?= $(HOSTCFLAGS)
+CXXFLAGS ?= $(CFLAGS)
 LDFLAGS ?= $(HOSTLDFLAGS)
 
-export PKG_CONFIG
-export HOST_PKG_CONFIG
+NINJA ?= ninja
 
 ifdef VERBOSE
 	hide =
@@ -63,37 +64,33 @@ EXT ?=
 
 CWD=$(shell pwd)
 
-ifeq ($(AB_ENABLE_PROGRESS_INFO),true)
-	ifeq ($(PROGRESSINFO),)
-	# The first make invocation here has to have its output discarded or else it
-	# produces spurious 'Leaving directory' messages... don't know why.
-	rulecount := $(strip $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 > /dev/null \
-		&& $(MAKE) --no-print-directory -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l))
-	ruleindex := 1
-	PROGRESSINFO = "[$(ruleindex)/$(rulecount)]$(eval ruleindex := $(shell expr $(ruleindex) + 1)) "
-	endif
-else
-	PROGRESSINFO = ""
-endif
+define newline
 
-PKG_CONFIG_HASHES = $(OBJ)/.pkg-config-hashes/target-$(word 1, $(shell $(PKG_CONFIG) --list-all | md5sum))
-HOST_PKG_CONFIG_HASHES = $(OBJ)/.pkg-config-hashes/host-$(word 1, $(shell $(HOST_PKG_CONFIG) --list-all | md5sum))
 
-$(OBJ)/build.mk : $(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES)
-$(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES) &:
-	$(hide) rm -rf $(OBJ)/.pkg-config-hashes
-	$(hide) mkdir -p $(OBJ)/.pkg-config-hashes
-	$(hide) touch $(PKG_CONFIG_HASHES) $(HOST_PKG_CONFIG_HASHES)
+endef
 
-include $(OBJ)/build.mk
+define check_for_command
+  $(shell command -v $1 >/dev/null || (echo "Required command '$1' missing" >/dev/stderr && kill $$PPID))
+endef
 
-ifeq ($(OSX),yes)
-	MAKEFLAGS += -r -j$(shell sysctl -n hw.logicalcpu)
-else
-	MAKEFLAGS += -r -j$(shell nproc)
-endif
+$(call check_for_command,ninja)
+$(call check_for_command,cmp)
+$(call check_for_command,$(PYTHON))
 
-.DELETE_ON_ERROR:
+pkg-config-hash = $(shell ($(PKG_CONFIG) --list-all && $(HOST_PKG_CONFIG) --list-all) | md5sum)
+build-files = $(shell find . -name .obj -prune -o \( -name 'build.py' -a -type f \) -print) $(wildcard build/*.py) $(wildcard config.py)
+build-file-timestamps = $(shell ls -l $(build-files) | md5sum)
+
+# Wipe the build file (forcing a regeneration) if the make environment is different.
+# (Conveniently, this includes the pkg-config hash calculated above.)
+
+ignored-variables = MAKE_RESTARTS .VARIABLES MAKECMDGOALS MAKEFLAGS MFLAGS
+$(shell mkdir -p $(OBJ))
+$(file >$(OBJ)/newvars.txt,$(foreach v,$(filter-out $(ignored-variables),$(.VARIABLES)),$(v)=$($(v))$(newline)))
+$(shell touch $(OBJ)/vars.txt)
+#$(shell diff -u $(OBJ)/vars.txt $(OBJ)/newvars.txt > /dev/stderr)
+$(shell cmp -s $(OBJ)/newvars.txt $(OBJ)/vars.txt || (rm -f $(OBJ)/build.ninja && echo "Environment changed --- regenerating" > /dev/stderr))
+$(shell mv $(OBJ)/newvars.txt $(OBJ)/vars.txt)
 
 .PHONY: update-ab
 update-ab:
@@ -108,9 +105,15 @@ clean::
 	$(hide) rm -rf $(OBJ)
 
 export PYTHONHASHSEED = 1
-build-files = $(shell find . -name 'build.py') $(wildcard build/*.py) $(wildcard config.py)
-$(OBJ)/build.mk: Makefile $(build-files) build/ab.mk
+$(OBJ)/build.ninja $(OBJ)/build.targets &:
 	@echo "AB"
-	@mkdir -p $(OBJ)
-	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py -o $@ build.py \
-		|| rm -f $@
+	$(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py \
+		-o $(OBJ) build.py \
+		-v $(OBJ)/vars.txt \
+		|| (rm -f $@ && false)
+
+include $(OBJ)/build.targets
+.PHONY: $(ninja-targets)
+.NOTPARALLEL:
+$(ninja-targets): $(OBJ)/build.ninja
+	+$(hide) $(NINJA) -f $(OBJ)/build.ninja $@

build/ab.ninja

@@ -0,0 +1,2 @@
+rule rule
+    command = $command

build/ab.py

@@ -1,36 +1,32 @@
+from collections import namedtuple
+from copy import copy
+from importlib.machinery import SourceFileLoader, PathFinder, ModuleSpec
 from os.path import *
 from pathlib import Path
 from typing import Iterable
 import argparse
+import ast
 import builtins
-from copy import copy
 import functools
+import hashlib
 import importlib
 import importlib.util
-from importlib.machinery import (
-    SourceFileLoader,
-    PathFinder,
-    ModuleSpec,
-)
 import inspect
+import os
+import re
 import string
 import sys
-import hashlib
-import re
-import ast
-from collections import namedtuple
+import types
 
-VERBOSE_MK_FILE = False
+VERBOSE_NINJA_FILE = False
 
-verbose = False
 quiet = False
 cwdStack = [""]
 targets = {}
 unmaterialisedTargets = {}  # dict, not set, to get consistent ordering
 materialisingStack = []
 defaultGlobals = {}
-globalId = 1
-wordCache = {}
+outputTargets = set()
 
 RE_FORMAT_SPEC = re.compile(
     r"(?:(?P<fill>[\s\S])?(?P<align>[<>=^]))?"
@@ -52,6 +48,15 @@ sys.path += ["."]
 old_import = builtins.__import__
 
 
+class Environment(types.SimpleNamespace):
+    def setdefault(self, name, value):
+        if not hasattr(self, name):
+            setattr(self, name, value)
+
+
+G = Environment()
+
+
 class PathFinderImpl(PathFinder):
     def find_spec(self, fullname, path, target=None):
         # The second test here is needed for Python 3.9.
@@ -102,27 +107,88 @@ def error(message):
     raise ABException(message)
 
 
+def _undo_escaped_dollar(s, op):
+    return s.replace(f"$${op}", f"${op}")
+
+
 class BracketedFormatter(string.Formatter):
     def parse(self, format_string):
         while format_string:
-            left, *right = format_string.split("$[", 1)
-            if not right:
-                yield (left, None, None, None)
+            m = re.search(f"(?:[^$]|^)()\\$\\[()", format_string)
+            if not m:
+                yield (
+                    _undo_escaped_dollar(format_string, "["),
+                    None,
+                    None,
+                    None,
+                )
                 break
-            right = right[0]
+            left = format_string[: m.start(1)]
+            right = format_string[m.end(2) :]
 
             offset = len(right) + 1
             try:
                 ast.parse(right)
             except SyntaxError as e:
-                if not str(e).startswith("unmatched ']'"):
+                if not str(e).startswith(f"unmatched ']'"):
                     raise e
                 offset = e.offset
 
             expr = right[0 : offset - 1]
             format_string = right[offset:]
 
-            yield (left if left else None, expr, None, None)
+            yield (
+                _undo_escaped_dollar(left, "[") if left else None,
+                expr,
+                None,
+                None,
+            )
+
+
+class GlobalFormatter(string.Formatter):
+    def parse(self, format_string):
+        while format_string:
+            m = re.search(f"(?:[^$]|^)()\\$\\(([^)]*)\\)()", format_string)
+            if not m:
+                yield (
+                    format_string,
+                    None,
+                    None,
+                    None,
+                )
+                break
+            left = format_string[: m.start(1)]
+            var = m[2]
+            format_string = format_string[m.end(3) :]
+
+            yield (
+                left if left else None,
+                var,
+                None,
+                None,
+            )
+
+    def get_field(self, name, a1, a2):
+        return (
+            getattr(G, name),
+            False,
+        )
+
+    def format_field(self, value, format_spec):
+        if not value:
+            return ""
+        return str(value)
+
+
+globalFormatter = GlobalFormatter()
+
+
+def substituteGlobalVariables(value):
+    while True:
+        oldValue = value
+        value = globalFormatter.format(value)
+        if value == oldValue:
+            return _undo_escaped_dollar(value, "(")
 
 
 def Rule(func):
@@ -187,12 +253,10 @@ def _isiterable(xs):
 
 class Target:
     def __init__(self, cwd, name):
-        if verbose:
-            print("rule('%s', cwd='%s'" % (name, cwd))
         self.name = name
         self.localname = self.name.rsplit("+")[-1]
         self.traits = set()
-        self.dir = join("$(OBJ)", name)
+        self.dir = join(G.OBJ, name)
         self.ins = []
         self.outs = []
         self.deps = []
@@ -232,7 +296,8 @@ class Target:
                     [selfi.templateexpand(f) for f in filenamesof(value)]
                 )
 
-        return Formatter().format(s)
+        s = Formatter().format(s)
+        return substituteGlobalVariables(s)
 
     def materialise(self, replacing=False):
         if self not in unmaterialisedTargets:
@@ -341,10 +406,10 @@ def targetof(value, cwd=None):
             elif value.startswith("./"):
                 value = normpath(join(cwd, value))
         # Explicit directories are always raw files.
-        elif value.endswith("/"):
+        if value.endswith("/"):
             return _filetarget(value, cwd)
-        # Anything starting with a variable expansion is always a raw file.
-        elif value.startswith("$"):
+        # Anything in .obj is a raw file.
+        elif value.startswith(outputdir) or value.startswith(G.OBJ):
             return _filetarget(value, cwd)
 
         # If this is not a rule lookup...
@@ -467,78 +532,71 @@ def emit(*args, into=None):
     if into is not None:
         into += [s]
     else:
-        outputFp.write(s)
+        ninjaFp.write(s)
+
+
+def shell(*args):
+    s = "".join(args) + "\n"
+    shellFp.write(s)
 
 
 def emit_rule(self, ins, outs, cmds=[], label=None):
     name = self.name
-    fins_list = filenamesof(ins)
-    fins = set(fins_list)
-    fouts = filenamesof(outs)
-    nonobjs = [f for f in fouts if not f.startswith("$(OBJ)")]
+    fins = [self.templateexpand(f) for f in set(filenamesof(ins))]
+    fouts = [self.templateexpand(f) for f in filenamesof(outs)]
+
+    global outputTargets
+    outputTargets.update(fouts)
+    outputTargets.add(name)
 
     emit("")
-    if VERBOSE_MK_FILE:
+    if VERBOSE_NINJA_FILE:
         for k, v in self.args.items():
             emit(f"# {k} = {v}")
 
-    lines = []
-    if nonobjs:
-        emit("clean::", into=lines)
-        emit("\t$(hide) rm -f", *nonobjs, into=lines)
-
-    hashable = cmds + fins_list + fouts
-    hash = hashlib.sha1(bytes("\n".join(hashable), "utf-8")).hexdigest()
-    hashfile = join(self.dir, f"hash_{hash}")
-
-    global globalId
-    emit(".PHONY:", name, into=lines)
     if outs:
-        outsn = globalId
-        globalId = globalId + 1
-        insn = globalId
-        globalId = globalId + 1
-
-        emit(f"OUTS_{outsn}", "=", *fouts, into=lines)
-        emit(f"INS_{insn}", "=", *fins, into=lines)
-        emit(name, ":", f"$(OUTS_{outsn})", into=lines)
-        emit(hashfile, ":", into=lines)
-        emit(f"\t@mkdir -p {self.dir}", into=lines)
-        emit(f"\t@touch {hashfile}", into=lines)
-        emit(
-            f"$(OUTS_{outsn})",
-            "&:" if len(fouts) > 1 else ":",
-            f"$(INS_{insn})",
-            hashfile,
-            into=lines,
-        )
-
-        if label:
-            emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO)" + label, into=lines)
+        os.makedirs(self.dir, exist_ok=True)
+        rule = []
 
         sandbox = join(self.dir, "sandbox")
-        emit("\t$(hide)", f"rm -rf {sandbox}", into=lines)
+        emit(f"rm -rf {sandbox}", into=rule)
         emit(
-            "\t$(hide)",
-            "$(PYTHON) build/_sandbox.py --link -s",
-            sandbox,
-            f"$(INS_{insn})",
-            into=lines,
+            f"{G.PYTHON} build/_sandbox.py --link -s", sandbox, *fins, into=rule
         )
         for c in cmds:
-            emit(f"\t$(hide) cd {sandbox} && (", c, ")", into=lines)
+            emit(f"(cd {sandbox} &&", c, ")", into=rule)
         emit(
-            "\t$(hide)",
-            "$(PYTHON) build/_sandbox.py --export -s",
+            f"{G.PYTHON} build/_sandbox.py --export -s",
             sandbox,
-            f"$(OUTS_{outsn})",
-            into=lines,
+            *fouts,
+            into=rule,
         )
+
+        ruletext = "".join(rule)
+        if len(ruletext) > 7000:
+            rulehash = hashlib.sha1(ruletext.encode()).hexdigest()
+
+            rulef = join(self.dir, f"rule-{rulehash}.sh")
+            with open(rulef, "wt") as fp:
+                fp.write("set -e\n")
+                fp.write(ruletext)
+
+            emit("build", *fouts, ":rule", *fins, rulef)
+            emit(" command=sh", rulef)
+        else:
+            emit("build", *fouts, ":rule", *fins)
+            emit(
+                " command=",
+                "&&".join([s.strip() for s in rule]).replace("$", "$$"),
+            )
+        if label:
+            emit(" description=", label)
+        emit("build", name, ":phony", *fouts)
+
     else:
         assert len(cmds) == 0, "rules with no outputs cannot have commands"
-        emit(name, ":", *fins, into=lines)
+        emit("build", name, ":phony", *fins)
 
-    outputFp.write("".join(lines))
     emit("")
 
 
@@ -585,47 +643,65 @@ def export(self, name=None, items: TargetsMap = {}, deps: Targets = []):
         dest = self.targetof(dest)
         outs += [dest]
 
-        destf = filenameof(dest)
+        destf = self.templateexpand(filenameof(dest))
+        outputTargets.update([destf])
 
         srcs = filenamesof([src])
         assert (
             len(srcs) == 1
         ), "a dependency of an exported file must have exactly one output file"
+        srcf = self.templateexpand(srcs[0])
 
         subrule = simplerule(
             name=f"{self.localname}/{destf}",
             cwd=self.cwd,
             ins=[srcs[0]],
             outs=[destf],
-            commands=["$(CP) -H %s %s" % (srcs[0], destf)],
-            label="",
+            commands=["$(CP) -H %s %s" % (srcf, destf)],
+            label="EXPORT",
         )
         subrule.materialise()
 
     self.ins = []
     self.outs = deps + outs
+    outputTargets.add(name)
 
     emit("")
-    emit(".PHONY:", name)
-    emit(name, ":", *filenamesof(outs + deps))
+    emit(
+        "build",
+        name,
+        ":phony",
+        *[self.templateexpand(f) for f in filenamesof(outs + deps)],
+    )
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-v", "--verbose", action="store_true")
     parser.add_argument("-q", "--quiet", action="store_true")
-    parser.add_argument("-o", "--output")
+    parser.add_argument("-v", "--varfile")
+    parser.add_argument("-o", "--outputdir")
+    parser.add_argument("-D", "--define", action="append", default=[])
     parser.add_argument("files", nargs="+")
     args = parser.parse_args()
 
-    global verbose
-    verbose = args.verbose
-
     global quiet
     quiet = args.quiet
 
-    global outputFp
-    outputFp = open(args.output, "wt")
+    vardefs = args.define
+    if args.varfile:
+        with open(args.varfile, "rt") as fp:
+            vardefs = vardefs + list(fp)
+
+    for line in vardefs:
+        if "=" in line:
+            name, value = line.split("=", 1)
+            G.setdefault(name.strip(), value.strip())
+
+    global ninjaFp, shellFp, outputdir
+    outputdir = args.outputdir
+    G.setdefault("OBJ", outputdir)
+    ninjaFp = open(outputdir + "/build.ninja", "wt")
+    ninjaFp.write(f"include build/ab.ninja\n")
 
     for k in ["Rule"]:
         defaultGlobals[k] = globals()[k]
@@ -640,7 +716,10 @@ def main():
     while unmaterialisedTargets:
         t = next(iter(unmaterialisedTargets))
         t.materialise()
-    emit("AB_LOADED = 1\n")
+
+    with open(outputdir + "/build.targets", "wt") as fp:
+        fp.write("ninja-targets =")
+        fp.write(substituteGlobalVariables(" ".join(outputTargets)))
 
 
 main()

build/c.py

@@ -7,23 +7,22 @@ from build.ab import (
     flatten,
     simplerule,
     emit,
+    G,
 )
-from build.utils import filenamesmatchingof, stripext, collectattrs
+from build.utils import stripext, collectattrs
 from build.toolchain import Toolchain, HostToolchain
 from os.path import *
 
-emit(
-    """
-ifeq ($(OSX),no)
-STARTGROUP ?= -Wl,--start-group
-ENDGROUP ?= -Wl,--end-group
-endif
-"""
-)
+if G.OSX != "yes":
+    G.STARTGROUP = "-Wl,--start-group"
+    G.ENDGROUP = "-Wl,--end-group"
+else:
+    G.STARTGROUP = ""
+    G.ENDGROUP = ""
 
 Toolchain.CC = ["$(CC) -c -o $[outs[0]] $[ins[0]] $(CFLAGS) $[cflags]"]
 Toolchain.CPP = ["$(CC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
-Toolchain.CXX = ["$(CXX) -c -o $[outs[0]] $[ins[0]] $(CFLAGS) $[cflags]"]
+Toolchain.CXX = ["$(CXX) -c -o $[outs[0]] $[ins[0]] $(CXXFLAGS) $[cflags]"]
 Toolchain.AR = ["$(AR) cqs $[outs[0]] $[ins]"]
 Toolchain.ARXX = ["$(AR) cqs $[outs[0]] $[ins]"]
 Toolchain.CLINK = [
@@ -70,13 +69,9 @@ def _toolchain_find_header_targets(deps, initial=[]):
 Toolchain.find_c_header_targets = _toolchain_find_header_targets
 
 
-HostToolchain.CC = [
-    "$(HOSTCC) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"
-]
+HostToolchain.CC = ["$(HOSTCC) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
 HostToolchain.CPP = ["$(HOSTCC) -E -P -o $[outs] $[cflags] -x c $[ins]"]
-HostToolchain.CXX = [
-    "$(HOSTCXX) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"
-]
+HostToolchain.CXX = ["$(HOSTCXX) -c -o $[outs[0]] $[ins[0]] $(HOSTCFLAGS) $[cflags]"]
 HostToolchain.AR = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
 HostToolchain.ARXX = ["$(HOSTAR) cqs $[outs[0]] $[ins]"]
 HostToolchain.CLINK = [
@@ -102,9 +97,7 @@ def _indirect(deps, name):
     return r
 
 
-def cfileimpl(
-    self, name, srcs, deps, suffix, commands, label, toolchain, cflags
-):
+def cfileimpl(self, name, srcs, deps, suffix, commands, label, toolchain, cflags):
     outleaf = "=" + stripext(basename(filenameof(srcs[0]))) + suffix
 
     hdr_deps = toolchain.find_c_header_targets(deps)
@@ -114,9 +107,7 @@ def cfileimpl(
         if ("cheader_deps" not in d.args) and ("clibrary_deps" not in d.args)
     ]
     hdr_files = collectattrs(targets=hdr_deps, name="cheader_files")
-    cflags = collectattrs(
-        targets=hdr_deps, name="caller_cflags", initial=cflags
-    )
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
 
     t = simplerule(
         replaces=self,
@@ -194,7 +185,7 @@ def findsources(self, srcs, deps, cflags, filerule, toolchain, cwd):
     for s in flatten(srcs):
         objs += [
             filerule(
-                name=join(self.localname, _removeprefix(f, "$(OBJ)/")),
+                name=join(self.localname, _removeprefix(f, G.OBJ + "/")),
                 srcs=[f],
                 deps=deps,
                 cflags=sorted(set(cflags)),
@@ -239,9 +230,7 @@ def libraryimpl(
         i = 0
         for dest, src in hdrs.items():
             s = filenamesof([src])
-            assert (
-                len(s) == 1
-            ), "the target of a header must return exactly one file"
+            assert len(s) == 1, "the target of a header must return exactly one file"
 
             cs += [f"$(CP) $[ins[{i}]] $[outs[{i}]]"]
             outs += ["=" + dest]
@@ -431,15 +420,11 @@ def programimpl(
     label,
     filerule,
 ):
-    cfiles = findsources(
-        self, srcs, deps, cflags, filerule, toolchain, self.cwd
-    )
+    cfiles = findsources(self, srcs, deps, cflags, filerule, toolchain, self.cwd)
 
     lib_deps = toolchain.find_c_library_targets(deps)
     libs = collectattrs(targets=lib_deps, name="clibrary_files")
-    ldflags = collectattrs(
-        targets=lib_deps, name="caller_ldflags", initial=ldflags
-    )
+    ldflags = collectattrs(targets=lib_deps, name="caller_ldflags", initial=ldflags)
 
     simplerule(
         replaces=self,
@@ -558,9 +543,7 @@ def hostcxxprogram(
 
 def _cppfileimpl(self, name, srcs, deps, cflags, toolchain):
     hdr_deps = _indirect(deps, "cheader_deps")
-    cflags = collectattrs(
-        targets=hdr_deps, name="caller_cflags", initial=cflags
-    )
+    cflags = collectattrs(targets=hdr_deps, name="caller_cflags", initial=cflags)
 
     simplerule(
         replaces=self,

build/pkg.py

@@ -1,4 +1,4 @@
-from build.ab import Rule, Target
+from build.ab import Rule, Target, G
 import os
 import subprocess
 
@@ -31,8 +31,8 @@ class _PkgConfig:
         return self.package_properties[p]
 
 
-TargetPkgConfig = _PkgConfig(os.getenv("PKG_CONFIG"))
-HostPkgConfig = _PkgConfig(os.getenv("HOST_PKG_CONFIG"))
+TargetPkgConfig = _PkgConfig(G.PKG_CONFIG)
+HostPkgConfig = _PkgConfig(G.HOST_PKG_CONFIG)
 
 
 def _package(self, name, package, fallback, pkgconfig):
@@ -49,9 +49,7 @@ def _package(self, name, package, fallback, pkgconfig):
         self.traits.update({"clibrary", "cxxlibrary"})
         return
 
-    assert (
-        fallback
-    ), f"Required package '{package}' not installed when materialising target '$[name]'"
+    assert fallback, f"Required package '{package}' not installed"
 
     if "cheader_deps" in fallback.args:
         self.args["cheader_deps"] = fallback.args["cheader_deps"]

build/protobuf.py

@@ -1,14 +1,10 @@
-from build.ab import Rule, Targets, emit, simplerule, filenamesof
+from build.ab import Rule, Targets, emit, simplerule, filenamesof, G
 from build.utils import filenamesmatchingof, collectattrs
 from os.path import join, abspath, dirname, relpath
 from build.pkg import has_package
 
-emit(
-    """
-PROTOC ?= protoc
-HOSTPROTOC ?= protoc
-"""
-)
+G.setdefault("PROTOC", "protoc")
+G.setdefault("HOSTPROTOC", "hostprotoc")
 
 assert has_package("protobuf"), "required package 'protobuf' not installed"
 

build/utils.py

@@ -7,10 +7,13 @@ from build.ab import (
     cwdStack,
     error,
     simplerule,
+    G
 )
 from os.path import relpath, splitext, join, basename, isfile
 from glob import iglob
 import fnmatch
+import subprocess
+import shutil
 
 
 def filenamesmatchingof(xs, pattern):
@@ -51,6 +54,16 @@ def itemsof(pattern, root=None, cwd=None):
     return result
 
 
+def does_command_exist(cmd):
+    basecmd = cmd.strip().split()[0]
+    return shutil.which(basecmd)
+
+
+def shell(cmd):
+    r = subprocess.check_output([G.SHELL, "-c", cmd])
+    return r.decode("utf-8").strip()
+
+
 @Rule
 def objectify(self, name, src: Target, symbol):
     simplerule(

build/zip.py

@@ -7,9 +7,7 @@ from build.ab import (
 
 
 @Rule
-def zip(
-    self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP"
-):
+def zip(self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP"):
     cs = ["$(PYTHON) build/_zip.py -z $[outs]"]
 
     ins = []

dep/adflib/build.py

@@ -35,7 +35,7 @@ clibrary(
         "./config.h",
         "./src/adflib.h",
     ],
-    cflags=["-Idep/adflib", "-Idep/adflib/src"],
+    cflags=["-Wno-stringop-overflow"],
     hdrs={
         "adf_blk.h": "./src/adf_blk.h",
         "adf_defs.h": "./src/adf_defs.h",

doc/building.md

@@ -204,18 +204,18 @@ install some support packages.
   - For Linux with Ubuntu/Debian:
 	`libusb-1.0-0-dev`, `libsqlite3-dev`, `zlib1g-dev`,
 	`libudev-dev`, `protobuf-compiler`, `libwxgtk3.0-gtk3-dev`,
-	`libfmt-dev`, `python3`.
+	`libfmt-dev`, `python3`. `ninja-build`
   - For Linux with Fedora/Red Hat:
     `git`, `make`, `gcc`, `gcc-c++`, `xxd`, `protobuf-compiler`,
     `protobuf-devel`, `fmt-devel`, `systemd-devel`, `wxGTK3-devel`,
-    `libsqlite3x-devel`
+    `libsqlite3x-devel`, `ninja-build`
   - For OSX with Homebrew: `libusb`, `pkg-config`, `sqlite`,
-    `protobuf`, `truncate`, `wxwidgets`, `fmt`.
+    `protobuf`, `truncate`, `wxwidgets`, `fmt`. `ninja`
   - For Windows with WSL: `protobuf-c-compiler` `protobuf-devel` `fmt-devel`
   `systemd-devel` `sqlite-devel` `wxGTK-devel` `mingw32-gcc` `mingw32-gcc-c++`
   `mingw32-zlib-static` `mingw32-protobuf-static` `mingw32-sqlite-static`
   `mingw32-wxWidgets3-static` `mingw32-libpng-static` `mingw32-libjpeg-static`
-  `mingw32-libtiff-static` `mingw32-nsis png2ico`
+  `mingw32-libtiff-static` `mingw32-nsis png2ico` `ninja-build`
 
 These lists are not necessarily exhaustive --- please [get in
 touch](https://github.com/davidgiven/fluxengine/issues/new) if I've missed

extras/build.py

@@ -4,6 +4,7 @@ from build.c import clibrary
 from build.zip import zip
 from glob import glob
 from os.path import *
+import config
 
 icons = ["fluxfile", "hardware", "icon", "imagefile"]
 
@@ -17,37 +18,37 @@ clibrary(
     },
 )
 
-simplerule(
-    name="fluxengine_icns",
-    ins=["./icon.png"],
-    outs=["=fluxengine.icns"],
-    commands=[
-        "mkdir -p fluxengine.iconset",
-        "sips -z 64 64 $[ins[0]] --out fluxengine.iconset/icon_32x32@2x.png > /dev/null",
-        "iconutil -c icns -o $[outs[0]] fluxengine.iconset",
-    ],
-    label="ICONSET",
-)
-
-simplerule(
-    name="fluxengine_ico",
-    ins=["./icon.png"],
-    outs=["=fluxengine.ico"],
-    commands=["png2ico $[outs[0]] $[ins[0]]"],
-    label="MAKEICON",
-)
-
-template_files = [
-    f
-    for f in glob(
-        "**", recursive=True, root_dir="extras/FluxEngine.app.template"
+if config.osx:
+    simplerule(
+        name="fluxengine_icns",
+        ins=["./icon.png"],
+        outs=["=fluxengine.icns"],
+        commands=[
+            "mkdir -p fluxengine.iconset",
+            "sips -z 64 64 $[ins[0]] --out fluxengine.iconset/icon_32x32@2x.png > /dev/null",
+            "iconutil -c icns -o $[outs[0]] fluxengine.iconset",
+        ],
+        label="ICONSET",
+    )
+
+    template_files = [
+        f
+        for f in glob("**", recursive=True, root_dir="extras/FluxEngine.app.template")
+        if isfile(join("extras/FluxEngine.app.template", f))
+    ]
+    zip(
+        name="fluxengine_template",
+        items={
+            join("FluxEngine.app", k): join("extras/FluxEngine.app.template", k)
+            for k in template_files
+        },
+    )
+
+if config.windows:
+    simplerule(
+        name="fluxengine_ico",
+        ins=["./icon.png"],
+        outs=["=fluxengine.ico"],
+        commands=["png2ico $[outs[0]] $[ins[0]]"],
+        label="MAKEICON",
     )
-    if isfile(join("extras/FluxEngine.app.template", f))
-]
-zip(
-    name="fluxengine_template",
-    items={
-        join("FluxEngine.app", k): join("extras/FluxEngine.app.template", k)
-        for k in template_files
-    },
-)

scripts/protoencode.cc

@@ -4,11 +4,72 @@
 #include <fstream>
 #include "fmt/format.h"
 #include "lib/core/globals.h"
+#include "lib/core/logger.h"
 #include "tests/testproto.pb.h"
 #include "lib/config/config.pb.h"
 #include <sstream>
 #include <locale>
 
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const BeginSpeedOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const EndSpeedOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const BeginWriteOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const EndWriteOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const BeginReadOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const EndReadOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const TrackReadLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const DiskReadLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const BeginOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const EndOperationLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const OperationProgressLogMessage> m)
+{
+}
+
+void renderLogMessage(
+    LogRenderer& r, std::shared_ptr<const OptionLogMessage> m)
+{
+}
+
 const std::string protoname = STRINGIFY(PROTO);
 
 static uint32_t readu8(std::string::iterator& it, std::string::iterator end)

src/formats/juku.textpb

@@ -12,8 +12,10 @@ of a whole generation of Estonian IT professionals.
 
 The system uses dual 5.25 inch ИЗОТ ЕС5323 (IZOT ES5323)
 diskette drive with regular MFM encoded DSDD. The disks have
-a sector skew factor 2 and tracks start from outside of the
-diskette _for both sides_, which is a combination that somewhat
+a sector skew factor 2 and tracks are written on one side of
+the floppy until it is full and then continued on the other
+side, starting from the outside of the disk again. This differs
+from the most common alternating sides method and somewhat
 complicates reading CP/M filesystem content with common tools.
 
 Mostly 800kB (786kB) DSDD disks were used, but there are also
@@ -21,8 +23,8 @@ Mostly 800kB (786kB) DSDD disks were used, but there are also
 
 ## References (all in Estonian)
 
-  - [How to read the content of Juku disks?](https://github.com/infoaed/juku3000/blob/master/docs/kettad.md)
-  - [List of recovered Juku software](https://github.com/infoaed/juku3000/blob/master/docs/tarkvara-kataloog.md)
+  - [How to read/write Juku disk images?](https://j3k.infoaed.ee/kettad/)
+  - [List of recovered Juku software](https://j3k.infoaed.ee/tarkvara-kataloog/)
   - [System disks for E5104](https://elektroonikamuuseum.ee/juku_arvuti_tarkvara.html)
 >>>
 

src/gui/build.py

@@ -1,19 +1,19 @@
-from build.ab import emit, simplerule
+from build.ab import simplerule, G
 from build.c import cxxprogram
+from build.utils import shell, does_command_exist
 from glob import glob
 import config
 
-emit(
-    """
-WX_CONFIG ?= wx-config
-ifneq ($(strip $(shell command -v $(WX_CONFIG) >/dev/null 2>&1; echo $$?)),0)
-WX_CFLAGS = $(error Required binary 'wx-config' not found.)
-WX_LDFLAGS = $(error Required binary 'wx-config' not found.)
-else
-WX_CFLAGS := $(shell $(WX_CONFIG) --cxxflags base adv aui richtext core)
-WX_LDFLAGS := $(shell $(WX_CONFIG) --libs base adv aui richtext core)
-endif
-"""
+G.setdefault("WX_CONFIG", "wx-config")
+assert does_command_exist(G.WX_CONFIG), "Required binary 'wx-config' not found"
+
+G.setdefault(
+    "WX_CFLAGS",
+    shell(f"{G.WX_CONFIG} --cxxflags base adv aui richtext core"),
+)
+G.setdefault(
+    "WX_LDFLAGS",
+    shell(f"{G.WX_CONFIG} --libs base adv aui richtext core"),
 )
 
 extrasrcs = ["./layout.cpp"]

tests/build.py

@@ -64,11 +64,14 @@ export(
                     "+protobuf_lib",
                     "+protocol",
                     ".+test_proto_lib",
+                    "dep/alphanum",
                     "dep/snowhouse",
+                    "lib/algorithms",
                     "lib/config",
                     "lib/core",
+                    "lib/data",
                     "lib/fluxsource+proto_lib",
-                    "dep/alphanum",
+                    "src/formats",
                 ],
             ),
         )