Refactor a lot of the proto stuff.

arch/build.py

@@ -2,7 +2,7 @@ from build.c import cxxlibrary
 from build.protobuf import proto, protocc
 
 proto(
-    name="arch_proto",
+    name="proto",
     srcs=[
         "./aeslanier/aeslanier.proto",
         "./agat/agat.proto",
@@ -24,4 +24,7 @@ proto(
         "./victor9k/victor9k.proto",
         "./zilogmcz/zilogmcz.proto",
     ],
+    deps=["lib+common_proto"],
 )
+
+protocc(name="proto_lib", srcs=[".+proto"], deps=["lib+common_proto_lib"])

build.py

@@ -199,6 +199,7 @@ cxxlibrary(
         "dep/stb",
         "lib+config_proto_lib",
         "lib/core",
+        "lib/fluxsource+proto_lib",
     ],
 )
 

build/ab.py

@@ -148,6 +148,7 @@ class Target:
         self.dir = join("$(OBJ)", name)
         self.ins = []
         self.outs = []
+        self.deps = []
         self.materialised = False
         self.args = {}
 

build/c.py

@@ -7,12 +7,7 @@ from build.ab import (
     flatten,
     simplerule,
 )
-from build.utils import (
-    filenamesmatchingof,
-    stripext,
-    targetswithtraitsof,
-    collectattrs,
-)
+from build.utils import filenamesmatchingof, stripext, collectattrs
 from os.path import *
 
 
@@ -39,12 +34,17 @@ class HostToolchain:
 def cfileimpl(self, name, srcs, deps, suffix, commands, label, kind, cflags):
     outleaf = "=" + stripext(basename(filenameof(srcs[0]))) + suffix
 
-    cflags = collectattrs(targets=deps, name="caller_cflags", initial=cflags)
+    hdr_deps = set()
+    for d in deps:
+        hdr_deps.update(d.args.get("cheader_deps", {d}))
+    cflags = collectattrs(
+        targets=hdr_deps, name="caller_cflags", initial=cflags
+    )
 
     t = simplerule(
         replaces=self,
         ins=srcs,
-        deps=deps,
+        deps=hdr_deps,
         outs=[outleaf],
         label=label,
         commands=commands,
@@ -93,11 +93,9 @@ def cxxfile(
 
 
 def findsources(name, srcs, deps, cflags, toolchain, filerule, cwd):
-    headers = filenamesmatchingof(srcs, "*.h")
-    cflags = cflags + ["-I" + dirname(h) for h in headers]
-
-    for d in deps:
-        headers += d.args.get("cheaders", [d])
+    for f in filenamesof(srcs):
+        if f.endswith(".h") or f.endswith(".hh"):
+            cflags = cflags + [f"-I{dirname(f)}"]
 
     objs = []
     for s in flatten(srcs):
@@ -105,8 +103,8 @@ def findsources(name, srcs, deps, cflags, toolchain, filerule, cwd):
             filerule(
                 name=join(name, f.removeprefix("$(OBJ)/")),
                 srcs=[f],
-                deps=headers,
-                cflags=cflags,
+                deps=deps,
+                cflags=sorted(set(cflags)),
                 toolchain=toolchain,
                 cwd=cwd,
             )
@@ -127,18 +125,10 @@ def findsources(name, srcs, deps, cflags, toolchain, filerule, cwd):
 def cheaders(
     self,
     name,
-    hdrs: TargetsMap = None,
+    hdrs: TargetsMap = {},
     caller_cflags=[],
-    deps: Targets = None,
+    deps: Targets = [],
 ):
-    hdr_deps = []
-    for d in deps:
-        hdr_deps += d.args.get("cheaders", [d])
-
-    hdr_caller_cflags = collectattrs(
-        targets=hdr_deps, name="caller_cflags", initial=caller_cflags
-    )
-
     cs = []
     ins = hdrs.values()
     outs = []
@@ -153,14 +143,24 @@ def cheaders(
         outs += ["=" + dest]
         i = i + 1
 
-    r = simplerule(
+    hdr_deps = {self}
+    lib_deps = set()
+    for d in deps:
+        hdr_deps.update(d.args.get("cheader_deps", {d}))
+        lib_deps.update(d.args.get("clibrary_deps", {d}))
+
+    simplerule(
         replaces=self,
         ins=ins,
         outs=outs,
+        deps=deps,
         commands=cs,
-        deps=hdr_deps,
         label="CHEADERS",
-        args={"caller_cflags": hdr_caller_cflags + ["-I" + self.dir]},
+        args={
+            "caller_cflags": caller_cflags + [f"-I{self.dir}"],
+            "cheader_deps": hdr_deps,
+            "clibrary_deps": lib_deps,
+        },
     )
 
 
@@ -184,7 +184,6 @@ def libraryimpl(
         cheaders(
             replaces=self,
             hdrs=hdrs,
-            deps=deps,
             caller_cflags=caller_cflags,
         )
         return
@@ -192,7 +191,6 @@ def libraryimpl(
         hr = cheaders(
             name=self.localname + "_hdrs",
             hdrs=hdrs,
-            deps=deps,
             caller_cflags=caller_cflags,
         )
         hr.materialise()
@@ -208,6 +206,12 @@ def libraryimpl(
         self.cwd,
     )
 
+    hdr_deps = {hr} if hr else set()
+    lib_deps = {self}
+    for d in deps:
+        hdr_deps.update(d.args.get("cheader_deps", {d}))
+        lib_deps.update(d.args.get("clibrary_deps", {d}))
+
     simplerule(
         replaces=self,
         ins=objs,
@@ -215,18 +219,12 @@ def libraryimpl(
         label=label,
         commands=commands,
         args={
-            "caller_ldflags": collectattrs(
-                targets=deps, name="caller_ldflags", initial=caller_ldflags
-            ),
-        }
-        | (
-            {"cheaders": [hr], "caller_cflags": hr.args["caller_cflags"]}
-            if hr
-            else {}
-        ),
+            "caller_ldflags": caller_ldflags,
+            "cheader_deps": hdr_deps,
+            "clibrary_deps": lib_deps,
+        },
         traits={"cheaders"},
     )
-    self.outs = self.outs + (hr.outs if hr else [])
 
 
 @Rule
@@ -314,17 +312,25 @@ def programimpl(
     label,
     filerule,
     kind,
+    libkind,
 ):
-    ars = filenamesmatchingof(deps, "*.a")
-
     cfiles = findsources(
         self.localname, srcs, deps, cflags, toolchain, filerule, self.cwd
     )
+
+    lib_deps = set()
+    for d in deps:
+        lib_deps.update(d.args.get("clibrary_deps", {d}))
+    libs = sorted(filenamesmatchingof(lib_deps, "*.a"))
+    ldflags = collectattrs(
+        targets=lib_deps, name="caller_ldflags", initial=ldflags
+    )
+
     simplerule(
         replaces=self,
-        ins=cfiles + ars + ars,
+        ins=cfiles + libs + libs,
         outs=[f"={self.localname}$(EXT)"],
-        deps=deps,
+        deps=lib_deps,
         label=toolchain.label + label,
         commands=commands,
         args={
@@ -346,8 +352,6 @@ def cprogram(
     toolchain=Toolchain,
     commands=None,
     label="CLINK",
-    cfilerule=cfile,
-    cfilekind="cprogram",
 ):
     if not commands:
         commands = toolchain.cprogram
@@ -361,8 +365,9 @@ def cprogram(
         toolchain,
         commands,
         label,
-        cfilerule,
-        cfilekind,
+        cfile,
+        "cprogram",
+        "clibrary",
     )
 
 
@@ -392,4 +397,5 @@ def cxxprogram(
         label,
         cxxfile,
         "cxxprogram",
+        "cxxlibrary",
     )

build/protobuf.py

@@ -1,7 +1,7 @@
 from build.ab import Rule, Targets, emit, simplerule, filenamesof
 from build.utils import filenamesmatchingof, collectattrs
 from types import SimpleNamespace
-from os.path import join
+from os.path import join, abspath, dirname
 import build.pkg  # to get the protobuf package check
 
 emit(
@@ -14,18 +14,54 @@ endif
 )
 
 
+def _getprotodeps(deps):
+    r = set()
+    for d in deps:
+        r.update(d.args.get("protodeps", {d}))
+    return sorted(r)
+
+
 @Rule
 def proto(self, name, srcs: Targets = [], deps: Targets = []):
+    protodeps = _getprotodeps(deps)
+    descriptorlist = ":".join(
+        [abspath(f) for f in filenamesmatchingof(protodeps, "*.descriptor")]
+    )
+
+    dirs = sorted({"{dir}/" + dirname(f) for f in filenamesof(srcs)})
     simplerule(
         replaces=self,
         ins=srcs,
-        outs=[f"={name}.descriptor"],
-        deps=deps,
-        commands=[
-            "$(PROTOC) --include_source_info --descriptor_set_out={outs[0]} {ins}"
-        ],
+        outs=[f"={self.localname}.descriptor"],
+        deps=protodeps,
+        commands=(
+            ["mkdir -p " + (" ".join(dirs))]
+            + [f"$(CP) {f} {{dir}}/{f}" for f in filenamesof(srcs)]
+            + [
+                "cd {dir} && "
+                + (
+                    " ".join(
+                        [
+                            "$(PROTOC)",
+                            "--proto_path=.",
+                            "--include_source_info",
+                            f"--descriptor_set_out={self.localname}.descriptor",
+                        ]
+                        + (
+                            [f"--descriptor_set_in={descriptorlist}"]
+                            if descriptorlist
+                            else []
+                        )
+                        + ["{ins}"]
+                    )
+                )
+            ]
+        ),
         label="PROTO",
-        args={"protosrcs": filenamesof(srcs)},
+        args={
+            "protosrcs": filenamesof(srcs),
+            "protodeps": set(protodeps) | {self},
+        },
     )
 
 
@@ -40,16 +76,33 @@ def protocc(self, name, srcs: Targets = [], deps: Targets = []):
         cc = f.replace(".proto", ".pb.cc")
         h = f.replace(".proto", ".pb.h")
         protos += [f]
-        srcs += [f]
         outs += ["=" + cc, "=" + h]
 
+    protodeps = _getprotodeps(deps + srcs)
+    descriptorlist = ":".join(
+        [abspath(f) for f in filenamesmatchingof(protodeps, "*.descriptor")]
+    )
+
     r = simplerule(
         name=f"{self.localname}_srcs",
         cwd=self.cwd,
-        ins=protos,
+        ins=srcs,
         outs=outs,
-        deps=deps,
-        commands=["$(PROTOC) --cpp_out={dir} {ins}"],
+        deps=protodeps,
+        commands=[
+            "cd {dir} && "
+            + (
+                " ".join(
+                    [
+                        "$(PROTOC)",
+                        "--proto_path=.",
+                        "--cpp_out=.",
+                        f"--descriptor_set_in={descriptorlist}",
+                    ]
+                    + protos
+                )
+            )
+        ],
         label="PROTOCC",
     )
 
@@ -76,6 +129,10 @@ def protojava(self, name, srcs: Targets = [], deps: Targets = []):
         protos += [f]
         srcs += [f]
 
+    descriptorlist = ":".join(
+        [abspath(f) for f in filenamesmatchingof(srcs + deps, "*.descriptor")]
+    )
+
     r = simplerule(
         name=f"{self.localname}_srcs",
         cwd=self.cwd,
@@ -84,7 +141,18 @@ def protojava(self, name, srcs: Targets = [], deps: Targets = []):
         deps=deps,
         commands=[
             "mkdir -p {dir}/srcs",
-            "$(PROTOC) --java_out={dir}/srcs {ins}",
+            "cd {dir} && "
+            + (
+                " ".join(
+                    [
+                        "$(PROTOC)",
+                        "--proto_path=.",
+                        "--java_out=.",
+                        f"--descriptor_set_in={descriptorlist}",
+                    ]
+                    + protos
+                )
+            ),
             "$(JAR) cf {outs[0]} -C {dir}/srcs .",
         ],
         traits={"srcjar"},

build/utils.py

@@ -30,7 +30,7 @@ def collectattrs(*, targets, name, initial=[]):
     s = set(initial)
     for a in [t.args.get(name, []) for t in targets]:
         s.update(a)
-    return sorted(list(s))
+    return sorted(s)
 
 
 def itemsof(pattern, root=None, cwd=None):

lib/build.py

@@ -1,28 +1,61 @@
-from build.c import cxxlibrary
 from build.protobuf import proto, protocc
 
 
 proto(name="common_proto", srcs=["./common.proto"])
+protocc(name="common_proto_lib", srcs=[".+common_proto"])
+
+proto(
+    name="layout_proto",
+    srcs=["./layout.proto"],
+    deps=[".+common_proto", "+fl2_proto"],
+)
+protocc(
+    name="layout_proto_lib",
+    srcs=[".+layout_proto"],
+    deps=[".+common_proto_lib", "+fl2_proto_lib"],
+)
+
+proto(
+    name="drive_proto",
+    srcs=["./drive.proto"],
+    deps=[".+common_proto", "+fl2_proto", ".+layout_proto"],
+)
+protocc(
+    name="drive_proto_lib",
+    srcs=[".+drive_proto"],
+    deps=[".+common_proto_lib", "+fl2_proto_lib", ".+layout_proto_lib"],
+)
 
 proto(
     name="config_proto",
     srcs=[
         "./config.proto",
-        "./layout.proto",
-        "./drive.proto",
         "./decoders/decoders.proto",
-        "./encoders/encoders.proto",
-        "./fluxsink/fluxsink.proto",
-        "./fluxsource/fluxsource.proto",
         "./imagereader/imagereader.proto",
         "./imagewriter/imagewriter.proto",
-        "./usb/usb.proto",
-        "./vfs/vfs.proto",
     ],
-    deps=[".+common_proto", "+fl2_proto"],
+    deps=[
+        ".+common_proto",
+        ".+layout_proto",
+        ".+drive_proto",
+        "+fl2_proto",
+        "lib/fluxsource+proto",
+        "lib/fluxsink+proto",
+        "lib/vfs+proto",
+        "lib/usb+proto",
+        "lib/encoders+proto",
+    ],
 )
 
 protocc(
     name="config_proto_lib",
-    srcs=[".+common_proto", ".+config_proto", "arch+arch_proto", "+fl2_proto"]
+    srcs=[".+common_proto", ".+config_proto", "arch+proto", "+fl2_proto"],
+    deps=[
+        "lib/fluxsource+proto_lib",
+        "lib/fluxsink+proto_lib",
+        "lib/vfs+proto_lib",
+        "lib/usb+proto_lib",
+        "lib/encoders+proto_lib",
+        "lib+drive_proto_lib",
+    ],
 )

lib/encoders/build.py

@@ -0,0 +1,13 @@
+from build.protobuf import proto, protocc
+
+
+proto(
+    name="proto",
+    srcs=["./encoders.proto"],
+    deps=["lib+common_proto", "arch+proto"],
+)
+protocc(
+    name="proto_lib",
+    srcs=[".+proto"],
+    deps=["lib+common_proto_lib", "arch+proto_lib"],
+)

lib/fluxsink/build.py

@@ -0,0 +1,5 @@
+from build.protobuf import proto, protocc
+
+proto(name="proto", srcs=["./fluxsink.proto"], deps=["lib+common_proto"])
+
+protocc(name="proto_lib", srcs=[".+proto"], deps=["lib+common_proto_lib"])

lib/fluxsource/build.py

@@ -0,0 +1,9 @@
+from build.protobuf import proto, protocc
+
+proto(name="proto", srcs=["./fluxsource.proto"], deps=["lib+common_proto"])
+
+protocc(
+    name="proto_lib",
+    srcs=[".+proto"],
+    deps=["lib+common_proto", "lib+common_proto_lib"],
+)

lib/vfs/build.py

@@ -1,4 +1,21 @@
 from build.c import cxxlibrary
+from build.protobuf import proto, protocc
+
+proto(
+    name="proto",
+    srcs=["./vfs.proto"],
+    deps=["lib+common_proto", "lib+layout_proto", "+fl2_proto"],
+)
+
+protocc(
+    name="proto_lib",
+    srcs=[".+proto"],
+    deps=[
+        "lib+common_proto_lib",
+        "lib+layout_proto_lib",
+        "+fl2_proto_lib",
+    ],
+)
 
 cxxlibrary(
     name="vfs",
@@ -31,5 +48,6 @@ cxxlibrary(
     deps=[
         "+lib",
         "+fmt_lib",
+        ".+proto_lib",
     ],
 )

scripts/build.py

@@ -13,6 +13,8 @@ def protoencode_single(self, name, srcs: Targets, proto, symbol):
             cflags=["-DPROTO=" + proto],
             deps=[
                 "lib+config_proto_lib",
+                "lib/fluxsource+proto_lib",
+                "lib/fluxsink+proto_lib",
                 "tests+test_proto_lib",
                 "+protobuf_lib",
                 "+fmt_lib",
@@ -61,6 +63,8 @@ cxxprogram(
     deps=[
         "src/formats",
         "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
+        "lib/fluxsink+proto_lib",
         "+lib",
         "+fmt_lib",
         "+protobuf_lib",
@@ -73,6 +77,8 @@ cxxprogram(
     deps=[
         "src/formats",
         "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
+        "lib/fluxsink+proto_lib",
         "+lib",
         "+fmt_lib",
         "+protobuf_lib",

src/build.py

@@ -45,6 +45,7 @@ cxxprogram(
         "lib/core",
         "lib/vfs",
         "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
         "src/formats",
     ],
 )

src/gui/build.py

@@ -65,6 +65,7 @@ cxxprogram(
         "lib/core",
         "lib/vfs",
         "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
         "src/formats",
         "src/gui/drivetypes",
         "+z_lib",

tests/build.py

@@ -5,10 +5,7 @@ from build.utils import test
 from scripts.build import protoencode_single
 
 
-proto(
-    name="test_proto",
-    srcs=["./testproto.proto"],
-)
+proto(name="test_proto", srcs=["./testproto.proto"], deps=["lib+common_proto"])
 
 protocc(
     name="test_proto_lib", srcs=[".+test_proto"], deps=["lib+config_proto_lib"]
@@ -59,6 +56,7 @@ export(
                     "+fmt_lib",
                     "+lib",
                     "lib/core",
+                    "lib/fluxsource+proto_lib",
                     "+protobuf_lib",
                     "+protocol",
                     "+z_lib",
@@ -87,6 +85,7 @@ export(
                     "+fmt_lib",
                     "+lib",
                     "lib/core",
+                    "lib/fluxsource+proto_lib",
                     "+protobuf_lib",
                     "+protocol",
                     "+z_lib",

tools/build.py

@@ -8,14 +8,28 @@ if config.windows:
 cxxprogram(
     name="brother120tool",
     srcs=["./brother120tool.cc"],
-    deps=["+lib", "lib/core", "lib+config_proto_lib", "+fmt_lib", "+z_lib"]
+    deps=[
+        "+lib",
+        "lib/core",
+        "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
+        "+fmt_lib",
+        "+z_lib",
+    ]
     + emu,
 )
 
 cxxprogram(
     name="brother240tool",
     srcs=["./brother240tool.cc"],
-    deps=["+lib", "lib/core", "lib+config_proto_lib", "+fmt_lib", "+z_lib"]
+    deps=[
+        "+lib",
+        "lib/core",
+        "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
+        "+fmt_lib",
+        "+z_lib",
+    ]
     + emu,
 )
 
@@ -32,6 +46,7 @@ cxxprogram(
         "+z_lib",
         "dep/libusbp",
         "lib+config_proto_lib",
+        "lib/fluxsource+proto_lib",
         "src/formats",
         "lib/core",
     ],