rules: Fix layout compat rules

Layout compatibility rules are broken in configurations with 2 or more
layouts, e.g.:

- Works:
  - config: `--layout fi --variant basic`
  - symbols: `pc+fi(classic)+inet(evdev)`.
    `fi(basic)` is correctly remapped to `fi(classic)`.
- Does not work:
  - config `--layout fi,us --variant basic,`
  - symbols: `pc+fi(basic)+us:2+inet(evdev)`.
    `fi(basic)` is not remapped and leads to an include error.
- Does not work:
  - config `--layout us,fi --variant ,basic`
  - symbols: `pc+us:fi(basic):2+fi(classic):2+inet(evdev)`.
    `fi(basic)` is not remapped and leads to an include error.

I initially thought merging the following rules sets to fix it:

1. `model layout[n] = symbols`
2. `model layout[n] variant[n] = symbols`

but it does not work: `*` wildcard does *not* match empty values.
There is another issue: using a variant will check and match *both*
rules sets. In the first rule set, compat rules are only for deleted
symbols files (e.g. `dev`). But when the symbols file exists, we end
up with an invalid include statement (see `fi(basic)` examples above).
This is because the first rules set has a catch-all rule that will
always match, while we want it to match only in the second rules set.

Unfortunately such rules cannot be expressed in rules files. So the
solution is to append generated compatibility sections in the
corresponding symbols files and remove the corresponding current rules.

Section name clashes are checked and will raise an error.
E.g.: currently we append the following to `symbols/de`:

    partial xkb_symbols "lld" {
        include "it(lldde)"
    };

If we already had a section named lld, this would produce the following
error:

    Cannot add compatibility section in symbols/de: "lld" already exists

Also:
- Move symbols handling in meson into the symbols subfolder.
- Sort compat layout rules.
- We ensure that rules with variant take priority over rules without one.

Part-of: <https://gitlab.freedesktop.org/xkeyboard-config/xkeyboard-config/-/merge_requests/728>

4 files changed, 195 insertions, 135 deletions

diff --git a/rules/0015-modellayout1_symbols.part b/rules/0016-modellayout1_symbols.part
index db5b62de..db5b62de 100644
--- a/rules/0015-modellayout1_symbols.part
+++ b/rules/0016-modellayout1_symbols.part
diff --git a/rules/compat/layoutsMapping.lst b/rules/compat/layoutsMapping.lst
index 54d9fb93..c8c64236 100644
--- a/rules/compat/layoutsMapping.lst
+++ b/rules/compat/layoutsMapping.lst
@@ -10,3 +10,4 @@ tel	in(tel)
 tml	in(tam)
 us_intl	us(alt-intl)
 mao	nz(mao)		// Delete in 2028.
+mal	in(mal)
diff --git a/rules/compat/map-variants.py b/rules/compat/map-variants.py
index a495e61c..dd24882b 100755
--- a/rules/compat/map-variants.py
+++ b/rules/compat/map-variants.py
@@ -1,21 +1,45 @@
 #!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
 
 import argparse
+import functools
+import itertools
 import re
 import sys
-
-
-class Layout(object):
-    PATTERN = re.compile(r"(?P<layout>[^(]+)\((?P<variant>[^)]+)\)")
-
-    def __init__(self, layout, variant=None):
-        if match := self.PATTERN.match(layout):
-            assert variant is None
-            # parse a layout(variant) string
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from string import Template
+from typing import ClassVar, Generator
+
+ROOT = Path(__file__).parent.parent.parent
+RULES = ROOT / "rules"
+SYMBOLS = ROOT / "symbols"
+# Some checks in case we move this script
+assert RULES.is_dir()
+assert SYMBOLS.is_dir()
+
+
+@functools.total_ordering
+@dataclass(frozen=True, order=False)
+class Layout:
+    PATTERN: ClassVar[re.Pattern[str]] = re.compile(
+        r"(?P<layout>[^(]+)\((?P<variant>[^)]+)\)"
+    )
+
+    layout: str
+    variant: str
+
+    @classmethod
+    def parse(cls, layout: str, variant: str = ""):
+        # parse a layout(variant) string
+        if match := cls.PATTERN.match(layout):
+            assert not variant
             layout = match.group("layout")
             variant = match.group("variant")
-        self.layout = layout
-        self.variant = variant
+        return cls(layout, variant)
 
     def __str__(self):
         if self.variant:
@@ -23,90 +47,59 @@ class Layout(object):
         else:
             return "{}".format(self.layout)
 
-
-def read_file(path):
-    """Returns a list of two-layout tuples [(layout1, layout2), ...]"""
-
-    layouts = []
-    for line in open(path):
-        # Remove optional comment
-        line = line.split("//")[0]
-        # Split on whitespaces
-        groups = tuple(line.split())
-        length = len(groups)
-        if length == 2:
-            l1 = Layout(groups[0])
-            l2 = Layout(groups[1])
-        elif length == 4:
-            l1 = Layout(groups[0], groups[1])
-            l2 = Layout(groups[2], groups[3])
+    def __lt__(self, other):
+        """
+        Custom compare function in order to deal with missing variant.
+        """
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        elif self.layout == other.layout:
+            if self.variant == other.variant:
+                return False
+            # Handle missing variant
+            elif self.variant and (not other.variant or other.variant == "*"):
+                return True
+            # Handle missing variant
+            elif (not self.variant or self.variant == "*") and other.variant:
+                return False
+            else:
+                return self.variant < other.variant
         else:
-            raise ValueError(f"Invalid line: {line}")
-        layouts.append((l1, l2))
-    return layouts
-
-
-# ml_s
-def write_fixed_layout(dest, mappings, write_header):
-    if write_header:
-        dest.write("! model		layout				=	symbols\n")
-    for l1, l2 in mappings:
-        if l1.variant:
-            raise ValueError(f"Unexpected variant: {l1}")
-        dest.write("  *		{}			=	pc+{}\n".format(l1, l2))
-
-
-# mln_s
-def write_layout_n(dest, mappings, number, write_header):
-    if write_header:
-        dest.write("! model		layout[{}]	=	symbols\n".format(number))
-
-    # symbols is one of
-    #   +layout(variant):2 ... where the map-to-layout has a proper variant
-    #   +layout%(v[2]):2 ... where the map-to-layout does not have a variant
-    # and where the number is 1, we have a base and drop the suffix, i.e.
-    # the above becomes
-    #   pc+layout(variant)
-    #   pc+layout%(v[1])
-
-    base = "pc" if number == 1 else ""
-    suffix = "" if number == 1 else ":{}".format(number)
-
-    for l1, l2 in mappings:
-        if l1.variant:
-            raise ValueError(f"Unexpected variant: {l1}")
-        second_layout = (
-            str(l2) if l2.variant else "{}%(v[{}])".format(l2.layout, number)
-        )
-        dest.write(
-            "  *		{}		=	{}+{}{}\n".format(
-                l1, base, second_layout, suffix
-            )
-        )
-
-
-# mlv_s
-def write_fixed_layout_variant(dest, mappings, write_header):
-    if write_header:
-        dest.write("! model		layout		variant		=	symbols\n")
-    for l1, l2 in mappings:
-        if not l1.variant:
-            raise ValueError(f"Expected variant: {l1}")
-        dest.write(
-            "  *		{}		{}		=	pc+{}\n".format(
-                l1.layout, l1.variant, l2
-            )
-        )
-
-
-# mlnvn_s
-def write_layout_n_variant_n(dest, mappings, number, write_header):
+            return self.layout < other.layout
+
+    @classmethod
+    def read_file(cls, path: str) -> Generator[tuple[Layout, Layout], None, None]:
+        """Returns a list of two-layout tuples [(layout1, layout2), ...]"""
+
+        with open(path, "rt", encoding="utf-8") as fd:
+            for line in fd:
+                # Remove optional comment
+                line = line.split("//")[0]
+                # Split on whitespaces
+                groups = tuple(line.split())
+                length = len(groups)
+                if length == 2:
+                    l1 = Layout.parse(groups[0])
+                    l2 = Layout.parse(groups[1])
+                elif length == 4:
+                    l1 = Layout.parse(groups[0], groups[1])
+                    l2 = Layout.parse(groups[2], groups[3])
+                else:
+                    raise ValueError(f"Invalid line: {line}")
+                yield (l1, l2)
+
+
+def write_rules(
+    dest,
+    mappings: list[tuple[Layout, Layout]],
+    number: int,
+    expect_variant: bool,
+    write_header: bool,
+):
+    index = f"[{number}]" if number > 0 else ""
     if write_header:
-        dest.write(
-            "! model		layout[{}]	variant[{}]	=	symbols\n".format(
-                number, number
-            )
-        )
+        variant = f"\t\tvariant{index}" if expect_variant else "\t\t"
+        dest.write(f"! model		layout{index}{variant}	=	symbols\n")
 
     # symbols is
     #   +layout(variant):2
@@ -115,56 +108,123 @@ def write_layout_n_variant_n(dest, mappings, number, write_header):
     #   pc+layout(variant)
     # This part is only executed for the variantMappings.lst
 
-    base = "pc" if number == 1 else ""
-    suffix = "" if number == 1 else ":{}".format(number)
+    base = "pc" if number <= 1 else ""
+    suffix = "" if number <= 1 else ":{}".format(number)
 
     for l1, l2 in mappings:
-        if not l1.variant:
-            raise ValueError(f"Expected variant: {l1}")
-        second_layout = (
-            str(l2) if l2.variant else "{}%(v[{}])".format(l2.layout, number)
-        )
+        if expect_variant ^ bool(l1.variant):
+            expectation = "Expected" if expect_variant else "Unexpected"
+            raise ValueError(f"{expectation} variant: {l1}")
+        variant = f"\t\t{l1.variant}" if expect_variant else "\t\t"
+        if l2.variant == "*":
+            second_layout = l2.layout
+        else:
+            second_layout = str(l2) if l2.variant else f"{l2.layout}%(v{index})"
         dest.write(
-            "  *		{}		{}	=	{}+{}{}\n".format(
-                l1.layout, l1.variant, base, second_layout, suffix
+            "  *		{}{}	=	{}+{}{}\n".format(
+                l1.layout, variant, base, second_layout, suffix
             )
         )
 
 
-def map_variant(dest, files, want="mls", number=None):
-    if number == 0:
-        number = None
-
-    for idx, f in enumerate(files):
-        write_header = idx == 0
-
-        mappings = read_file(f)
-        if want == "mls":
-            if number is None:
-                write_fixed_layout(dest, mappings, write_header)
-            else:
-                write_layout_n(dest, mappings, number, write_header)
-        elif want == "mlvs":
-            if number is None:
-                write_fixed_layout_variant(dest, mappings, write_header)
-            else:
-                write_layout_n_variant_n(dest, mappings, number, write_header)
-        else:
-            raise NotImplementedError()
+SYMBOLS_TEMPLATE = Template("""
+// Compatibility mapping
+partial xkb_symbols "${alias}" {
+    include "${target}"
+};
+""")
+
+
+def write_symbols(
+    dest: Path,
+    mappings: list[tuple[Layout, Layout]],
+    expect_variant: bool,
+    dry_run: bool,
+):
+    """
+    Append xkb_symbols entries
+    """
+    # Group by alias symbol file
+    files = defaultdict(list)
+    for mapping in mappings:
+        files[mapping[0].layout].append(mapping)
+
+    for filename, mappings in files.items():
+        src_path: Path = SYMBOLS / filename
+        # Get original file content
+        with src_path.open("rt", encoding="utf-8") as fd:
+            content = fd.read()
+        # Check that there is no clash with existing sections
+        new_sections = "|".join(re.escape(l1.variant) for l1, _ in mappings)
+        pattern = re.compile(rf'xkb_symbols\s+"(?P<section>{new_sections})"')
+        for line_number, line in enumerate(content.splitlines(), start=1):
+            # Drop comments
+            line = line.split("//")[0]
+            # Check for clashing section definition
+            if m := pattern.search(line):
+                l1 = mappings[0][0]
+                section = m.group("section")
+                raise ValueError(
+                    f'Cannot add compatibility section in symbols/{l1.layout}: "{section}" already exists at line {line_number}'
+                )
+        # Add compat sections
+        dest_path = dest / filename
+        # Print path to stdout, for path collection in meson
+        print(dest_path.name)
+        if dry_run:
+            continue
+        with dest_path.open("wt", encoding="utf-8") as fd:
+            fd.write(content)
+            for l1, l2 in mappings:
+                if expect_variant ^ bool(l1.variant):
+                    expectation = "Expected" if expect_variant else "Unexpected"
+                    raise ValueError(f"{expectation} variant: {l1}")
+                content = SYMBOLS_TEMPLATE.substitute(alias=l1.variant, target=l2)
+                fd.write(content)
+
+
+def check_symbols(filename: str) -> bool:
+    """
+    Check if symbols file exists
+    """
+    return (SYMBOLS / filename).is_file()
+
+
+def check_mapping(symbols: bool, mapping: tuple[Layout, Layout]) -> bool:
+    """
+    Check whether a mapping should be kept
+    """
+    return symbols == check_symbols(mapping[0].layout)
+
+
+def run(
+    dest: str, files, dry_run: bool, symbols: bool, expect_variant: bool, number: int
+):
+    check = functools.partial(check_mapping, symbols)
+    # Read all files and sort their entry on the aliased layout
+    mappings = sorted(
+        filter(check, itertools.chain.from_iterable(map(Layout.read_file, files))),
+        key=lambda ls: ls[0],
+    )
+    if symbols:
+        write_symbols(Path(dest), mappings, expect_variant, dry_run)
+    elif not dry_run:
+        fd = None
+        if dest == "-":
+            fd = sys.stdout
+        with fd or open(ns.dest, "w") as fd:
+            write_rules(fd, mappings, number, expect_variant, True)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser("variant mapping script")
-    parser.add_argument("--want", type=str, choices=["mls", "mlvs"])
-    parser.add_argument("--number", type=int, default=None)
+    parser.add_argument("--number", type=int, default=0)
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--has-variant", action="store_true")
+    parser.add_argument("--symbols", action="store_true", help="Write symbols files")
 
     parser.add_argument("dest", type=str)
     parser.add_argument("files", nargs="+", type=str)
     ns = parser.parse_args()
 
-    dest = None
-    if ns.dest == "-":
-        dest = sys.stdout
-
-    with dest or open(ns.dest, "w") as fd:
-        map_variant(fd, ns.files, ns.want, ns.number)
+    run(ns.dest, ns.files, ns.dry_run, ns.symbols, ns.has_variant, ns.number)
diff --git a/rules/meson.build b/rules/meson.build
index 121aa972..6144618a 100644
--- a/rules/meson.build
+++ b/rules/meson.build
@@ -19,7 +19,7 @@ parts = [
     '0009-model_geometry.part',
     '0011-modellayoutvariant_symbols.part',
     '0013-modellayout_symbols.part',
-    '0015-modellayout1_symbols.part',
+    '0016-modellayout1_symbols.part',
     '0018-modellayout2_symbols.part',
     '0020-modellayout3_symbols.part',
     '0022-modellayout4_symbols.part',
@@ -80,7 +80,7 @@ if get_option('compat-rules')
   # each with the level name in the filename
   lvl_ml_s = {
     '0': '0012-ml_s.part',
-    '1': '0014-ml1_s.part',
+    '1': '0015-ml1_s.part',
     '2': '0017-ml2_s.part',
     '3': '0019-ml3_s.part',
     '4': '0021-ml4_s.part',
@@ -88,7 +88,7 @@ if get_option('compat-rules')
 
   lvl_mlv_s = {
     '0': '0010-mlv_s.part',
-    '1': '0016-ml1v1_s.part',
+    '1': '0014-ml1v1_s.part',
     '2': '0023-ml2v2_s.part',
     '3': '0024-ml3v3_s.part',
     '4': '0025-ml4v4_s.part',
@@ -100,7 +100,6 @@ if get_option('compat-rules')
                            build_by_default: true,
                            command: [
                                map_variants_py,
-                               '--want=mls',
                                '--number=@0@'.format(lvl),
                                '@OUTPUT@',
                                layout_mappings,
@@ -114,7 +113,7 @@ if get_option('compat-rules')
                            build_by_default: true,
                            command: [
                                map_variants_py,
-                               '--want=mlvs',
+                               '--has-variant',
                                '--number=@0@'.format(lvl),
                                '@OUTPUT@',
                                variant_mappings,