X++: add extractor and QL library generated from the compiler's own AST

xpp/.gitignore

@@ -0,0 +1,4 @@
+extractor/**/bin/*
+extractor/**/obj/*
+ql/build/
+extractor-pack/

xpp/build-linux64.ps1

@@ -0,0 +1,25 @@
+param (
+    [Parameter(Mandatory=$true)][string]$cliFolder
+)
+
+# Mirrors the PowerShell extractor's build scripts: publish the extractor into the CLI's
+# language folder, then copy the extractor configuration, dbscheme and tool scripts alongside.
+#
+# The X++ compiler package is proprietary and is not bundled. The extractor resolves it at run
+# time from XPP_COMPILER_PACKAGE, which must be set wherever extraction happens.
+
+$toolsFolder = Join-Path (Join-Path (Join-Path $cliFolder "xpp") "tools") "linux64"
+# Only the extractor is shipped. Xpp.SchemaGenerator is a build-time tool and comes in
+# through the solution, so the project is published rather than the solution.
+dotnet publish (Join-Path "$PSScriptRoot/extractor" "Xpp.Extractor" | Resolve-Path) -o $toolsFolder -r linux-x64 -c Release --self-contained
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Build failed"
+    exit 1
+}
+
+$xppFolder = Join-Path -Path $cliFolder -ChildPath "xpp"
+Copy-Item -Path "$PSScriptRoot/codeql-extractor.yml" -Destination $xppFolder -Force
+$qlLibFolder = Join-Path -Path "$PSScriptRoot/ql" -ChildPath "lib"
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme") -Destination $xppFolder -Force
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme.stats") -Destination $xppFolder -Force
+Copy-Item -Path "$PSScriptRoot/tools" -Destination $xppFolder -Recurse -Force

xpp/build-osx64.ps1

@@ -0,0 +1,25 @@
+param (
+    [Parameter(Mandatory=$true)][string]$cliFolder
+)
+
+# Mirrors the PowerShell extractor's build scripts: publish the extractor into the CLI's
+# language folder, then copy the extractor configuration, dbscheme and tool scripts alongside.
+#
+# The X++ compiler package is proprietary and is not bundled. The extractor resolves it at run
+# time from XPP_COMPILER_PACKAGE, which must be set wherever extraction happens.
+
+$toolsFolder = Join-Path (Join-Path (Join-Path $cliFolder "xpp") "tools") "osx64"
+# Only the extractor is shipped. Xpp.SchemaGenerator is a build-time tool and comes in
+# through the solution, so the project is published rather than the solution.
+dotnet publish (Join-Path "$PSScriptRoot/extractor" "Xpp.Extractor" | Resolve-Path) -o $toolsFolder -r osx-x64 -c Release --self-contained
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Build failed"
+    exit 1
+}
+
+$xppFolder = Join-Path -Path $cliFolder -ChildPath "xpp"
+Copy-Item -Path "$PSScriptRoot/codeql-extractor.yml" -Destination $xppFolder -Force
+$qlLibFolder = Join-Path -Path "$PSScriptRoot/ql" -ChildPath "lib"
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme") -Destination $xppFolder -Force
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme.stats") -Destination $xppFolder -Force
+Copy-Item -Path "$PSScriptRoot/tools" -Destination $xppFolder -Recurse -Force

xpp/build-win64.ps1

@@ -0,0 +1,25 @@
+param (
+    [Parameter(Mandatory=$true)][string]$cliFolder
+)
+
+# Mirrors the PowerShell extractor's build scripts: publish the extractor into the CLI's
+# language folder, then copy the extractor configuration, dbscheme and tool scripts alongside.
+#
+# The X++ compiler package is proprietary and is not bundled. The extractor resolves it at run
+# time from XPP_COMPILER_PACKAGE, which must be set wherever extraction happens.
+
+$toolsFolder = Join-Path (Join-Path (Join-Path $cliFolder "xpp") "tools") "win64"
+# Only the extractor is shipped. Xpp.SchemaGenerator is a build-time tool and comes in
+# through the solution, so the project is published rather than the solution.
+dotnet publish (Join-Path "$PSScriptRoot/extractor" "Xpp.Extractor" | Resolve-Path) -o $toolsFolder -r win-x64 -c Release --self-contained
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Build failed"
+    exit 1
+}
+
+$xppFolder = Join-Path -Path $cliFolder -ChildPath "xpp"
+Copy-Item -Path "$PSScriptRoot/codeql-extractor.yml" -Destination $xppFolder -Force
+$qlLibFolder = Join-Path -Path "$PSScriptRoot/ql" -ChildPath "lib"
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme") -Destination $xppFolder -Force
+Copy-Item -Path (Join-Path $qlLibFolder "xpp.dbscheme.stats") -Destination $xppFolder -Force
+Copy-Item -Path "$PSScriptRoot/tools" -Destination $xppFolder -Recurse -Force

xpp/codegen.conf

@@ -0,0 +1,10 @@
+# configuration file for X++ code generation default options
+--schema=schema
+--generate=dbscheme,ql
+--dbscheme=ql/lib/xpp.dbscheme
+--ql-output=ql/lib/codeql/xpp/elements/internal/generated
+--ql-stub-output=ql/lib/codeql/xpp/elements
+# codegen requires a test output directory, but its per-node-type placeholders are not worth
+# committing; real tests live in ql/test/library-tests. This path is gitignored.
+--ql-test-output=ql/build/extractor-tests
+--script-name=xpp/tools/generate-schema.sh

xpp/codeql-extractor.yml

@@ -0,0 +1,12 @@
+name: "xpp"
+display_name: "X++"
+version: 0.0.1
+column_kind: "utf16"
+legacy_qltest_extraction: true
+build_modes:
+  - none
+file_types:
+  - name: xpp
+    display_name: X++ metadata objects
+    extensions:
+      - .xml

xpp/extractor/Xpp.Extraction/AstSequence.cs

@@ -0,0 +1,68 @@
+using System.Collections;
+
+namespace Xpp.Extraction;
+
+/// <summary>
+/// Flattens the collection shapes the X++ compiler uses for child nodes.
+/// </summary>
+public static class AstSequence
+{
+    /// <summary>
+    /// The elements of <paramref name="value"/>, with dictionary entries reduced to their values.
+    /// </summary>
+    /// <remarks>
+    /// The compiler holds some children in dictionaries keyed by name, such as a class's fields
+    /// and methods. Iterating one as a bare <see cref="IEnumerable"/> yields
+    /// <see cref="KeyValuePair{TKey,TValue}"/> rather than the child, so the entries have to be
+    /// unwrapped or those subtrees are labelled wrongly and never traversed.
+    /// </remarks>
+    public static IEnumerable<object> Elements(object? value)
+    {
+        switch (value)
+        {
+            case null:
+            case string:
+                yield break;
+
+            // Dictionary<,> and the compiler's LinkedDictionary<,> both implement the
+            // non-generic interface, which exposes the values directly.
+            case IDictionary dictionary:
+                foreach (var item in dictionary.Values)
+                {
+                    if (item is not null)
+                        yield return item;
+                }
+
+                yield break;
+
+            case IEnumerable sequence:
+                foreach (var item in sequence)
+                {
+                    var unwrapped = Unwrap(item);
+                    if (unwrapped is not null)
+                        yield return unwrapped;
+                }
+
+                yield break;
+        }
+    }
+
+    /// <summary>
+    /// The value of a key/value entry, or the item itself when it is not one.
+    /// </summary>
+    /// <remarks>
+    /// A dictionary typed only as <c>IEnumerable&lt;KeyValuePair&lt;,&gt;&gt;</c> does not reach
+    /// the <see cref="IDictionary"/> case above, so entries are also unwrapped here.
+    /// </remarks>
+    private static object? Unwrap(object? item)
+    {
+        if (item is null)
+            return null;
+
+        var type = item.GetType();
+        if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(KeyValuePair<,>))
+            return type.GetProperty("Value")?.GetValue(item);
+
+        return item;
+    }
+}

xpp/extractor/Xpp.Extraction/CompilerPackageResolver.cs

@@ -0,0 +1,57 @@
+using System.Reflection;
+using System.Runtime.Loader;
+
+namespace Xpp.Extraction;
+
+/// <summary>
+/// Resolves the X++ compiler assemblies from an extracted
+/// <c>Microsoft.Dynamics.AX.Platform.CompilerPackage</c> at run time.
+/// </summary>
+/// <remarks>
+/// The package is proprietary, so it is neither committed nor copied into build output. The
+/// extractor locates it through <c>XPP_COMPILER_PACKAGE</c> instead, which also keeps a single
+/// copy shared between the extractor and any other tool that needs it.
+/// </remarks>
+public static class CompilerPackageResolver
+{
+    public const string PackageVariable = "XPP_COMPILER_PACKAGE";
+
+    private static bool installed;
+
+    /// <summary>
+    /// Starts resolving compiler assemblies from <paramref name="packageDirectory"/>, or from
+    /// <c>XPP_COMPILER_PACKAGE</c> when it is null.
+    /// </summary>
+    /// <exception cref="DirectoryNotFoundException">
+    /// The package directory is unset or does not exist.
+    /// </exception>
+    public static void Install(string? packageDirectory = null)
+    {
+        if (installed)
+            return;
+
+        packageDirectory ??= Environment.GetEnvironmentVariable(PackageVariable);
+
+        if (string.IsNullOrEmpty(packageDirectory))
+        {
+            throw new DirectoryNotFoundException(
+                $"{PackageVariable} is not set. Point it at an extracted " +
+                "Microsoft.Dynamics.AX.Platform.CompilerPackage.");
+        }
+
+        if (!Directory.Exists(packageDirectory))
+        {
+            throw new DirectoryNotFoundException(
+                $"{PackageVariable} does not exist: {packageDirectory}");
+        }
+
+        var root = packageDirectory;
+        AssemblyLoadContext.Default.Resolving += (context, name) =>
+        {
+            var candidate = Path.Combine(root, name.Name + ".dll");
+            return File.Exists(candidate) ? context.LoadFromAssemblyPath(candidate) : null;
+        };
+
+        installed = true;
+    }
+}