From 02ca32ae0a5bc290918d2b2a3288e385b9cc6b11 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 19 Feb 2021 17:10:51 -0500 Subject: external & build dependences in src tree - external & build dependences boost licensed - ext_depends (external depends) - D-YAML - tinyendian - d2sqlite3 - imageformats - build_depends - dub2nix --- .../dub2nix/.github/workflows/main.yml | 31 +++ src/build_depends/dub2nix/.gitignore | 5 + src/build_depends/dub2nix/LICENSE | 21 ++ src/build_depends/dub2nix/README.md | 82 +++++++ src/build_depends/dub2nix/default.nix | 2 + src/build_depends/dub2nix/dub.json | 15 ++ src/build_depends/dub2nix/dub.selections.json | 7 + src/build_depends/dub2nix/dub.selections.nix | 26 ++ src/build_depends/dub2nix/dub2nix.nix | 17 ++ src/build_depends/dub2nix/mkDub.nix | 121 ++++++++++ src/build_depends/dub2nix/shell.nix | 16 ++ src/build_depends/dub2nix/src/dub2nix.d | 263 +++++++++++++++++++++ 12 files changed, 606 insertions(+) create mode 100644 src/build_depends/dub2nix/.github/workflows/main.yml create mode 100644 src/build_depends/dub2nix/.gitignore create mode 100644 src/build_depends/dub2nix/LICENSE create mode 100644 src/build_depends/dub2nix/README.md create mode 100644 src/build_depends/dub2nix/default.nix create mode 100644 src/build_depends/dub2nix/dub.json create mode 100644 src/build_depends/dub2nix/dub.selections.json create mode 100644 src/build_depends/dub2nix/dub.selections.nix create mode 100644 src/build_depends/dub2nix/dub2nix.nix create mode 100644 src/build_depends/dub2nix/mkDub.nix create mode 100644 src/build_depends/dub2nix/shell.nix create mode 100644 src/build_depends/dub2nix/src/dub2nix.d (limited to 'src/build_depends/dub2nix') diff --git a/src/build_depends/dub2nix/.github/workflows/main.yml b/src/build_depends/dub2nix/.github/workflows/main.yml new file mode 100644 index 0000000..fe9e10a --- /dev/null +++ b/src/build_depends/dub2nix/.github/workflows/main.yml @@ -0,0 +1,31 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Install Nix + uses: cachix/install-nix-action@v12 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Build in shell + run: nix-shell --run "dub test" diff --git a/src/build_depends/dub2nix/.gitignore b/src/build_depends/dub2nix/.gitignore new file mode 100644 index 0000000..19eefee --- /dev/null +++ b/src/build_depends/dub2nix/.gitignore @@ -0,0 +1,5 @@ +dub2nix +/.dub/ +/result +/bin +.envrc diff --git a/src/build_depends/dub2nix/LICENSE b/src/build_depends/dub2nix/LICENSE new file mode 100644 index 0000000..88e0035 --- /dev/null +++ b/src/build_depends/dub2nix/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Lionello Lunesu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/build_depends/dub2nix/README.md b/src/build_depends/dub2nix/README.md new file mode 100644 index 0000000..bfb8e3a --- /dev/null +++ b/src/build_depends/dub2nix/README.md @@ -0,0 +1,82 @@ +# dub2nix +CLI tool to create Nix expressions for D-lang Dub projects. + +![CI](https://github.com/lionello/dub2nix/workflows/CI/badge.svg) + +## Installation +Install with `nix-env`: +```sh +nix-env -if https://github.com/lionello/dub2nix/archive/master.tar.gz +``` +or add to your `shell.nix` and run `nix-shell`: +```nix +with import {}; +let + dub2nix-src = fetchTarball { + url = "https://github.com/lionello/dub2nix/archive/master.tar.gz"; + }; + dub2nix = (import dub2nix-src) { inherit pkgs; }; +in mkShell { + buildInputs = [ + dub2nix # dub dmd rdmd ldc etc.. + ]; +} +``` + +## Development +Do `git clone` and `nix-shell` to build with `dub`: +```sh +nix-shell +dub +``` +Alternatively, use `direnv`: +```sh +echo use nix >> .envrc +direnv allow +dub +``` + +## Usage +``` +Usage: dub2nix [OPTIONS] COMMAND + +Create Nix derivations for Dub package dependencies. + +Commands: + save Write Nix files for Dub project + +Options: +-i --input Path of selections JSON; defaults to ./dub.selections.json +-o --output Output Nix file for Dub project +-r --registry URL to Dub package registry; default http://code.dlang.org/packages/ +-d --deps-file Output Nix file with dependencies; defaults to ./dub.selections.nix +-h --help This help information. +``` +First, use `dub build` to generate the `dub.selections.json` for your Dub project. +Then, run `dub2nix save` to read the `dub.selections.json` in the current folder and write a new file `dub.selections.nix`. + +This `dub.selections.nix` is used in `mkDubDerivation` (from `mkDub.nix`) to create a new derivation for your Dub project: +```nix +{pkgs}: +with (import ./mkDub.nix { + inherit pkgs; +}); +mkDubDerivation { + version = "0.1.0"; # optional + src = ./.; +} +``` + +When your project has no dependencies at all, this will fail because `dub.selections.nix` is missing. Set `deps` to override the dependencies: +```nix +{pkgs}: +with (import ./mkDub.nix { + inherit pkgs; +}); +mkDubDerivation { + src = ./.; + deps = []; +} +``` + +Use the `--output` option to write a `.nix` file with a skeleton derivation for your dub project. This will also create the `mkDub.nix` file for importing into the derivation. diff --git a/src/build_depends/dub2nix/default.nix b/src/build_depends/dub2nix/default.nix new file mode 100644 index 0000000..d967504 --- /dev/null +++ b/src/build_depends/dub2nix/default.nix @@ -0,0 +1,2 @@ +{ pkgs ? import {} }: +pkgs.callPackage ./dub2nix.nix {} diff --git a/src/build_depends/dub2nix/dub.json b/src/build_depends/dub2nix/dub.json new file mode 100644 index 0000000..18ed81d --- /dev/null +++ b/src/build_depends/dub2nix/dub.json @@ -0,0 +1,15 @@ +{ + "name": "dub2nix", + "authors": [ + "Lionello Lunesu" + ], + "dependencies": { + "vibe-d:data": "*" + }, + "stringImportPaths": ["."], + "description": "Create Nix derivations for Dub package dependencies", + "targetPath": "./bin/", + "copyright": "Copyright © 2020, Lionello Lunesu", + "license": "MIT", + "targetType": "executable" +} diff --git a/src/build_depends/dub2nix/dub.selections.json b/src/build_depends/dub2nix/dub.selections.json new file mode 100644 index 0000000..de6fe4d --- /dev/null +++ b/src/build_depends/dub2nix/dub.selections.json @@ -0,0 +1,7 @@ +{ + "fileVersion": 1, + "versions": { + "stdx-allocator": "2.77.5", + "vibe-d": "0.8.5" + } +} diff --git a/src/build_depends/dub2nix/dub.selections.nix b/src/build_depends/dub2nix/dub.selections.nix new file mode 100644 index 0000000..26a9c05 --- /dev/null +++ b/src/build_depends/dub2nix/dub.selections.nix @@ -0,0 +1,26 @@ +# This file was generated by https://github.com/lionello/dub2nix v0.2.4 +[ { + fetch = { + type = "git"; + url = "https://github.com/vibe-d/vibe.d.git"; + rev = "v0.8.5"; + sha256 = "0s1caxqmq2497j5x8h06f44nr597h9zac8qxxml953lkaqkhbzgy"; + fetchSubmodules = false; + date = "2019-03-24T14:45:15+01:00"; + deepClone = false; + leaveDotGit = false; + path = "/nix/store/kz5g44dncvznlkm38a74cmm4qyl9nr6b-vibe.d"; + }; +} { + fetch = { + type = "git"; + url = "https://github.com/wilzbach/stdx-allocator.git"; + rev = "v2.77.5"; + sha256 = "03av8zp5p6vf6fg005xbmwnjfw96jyrr4dcj4m56c4a3vx7v72pk"; + fetchSubmodules = false; + date = "2018-12-23T13:54:22+01:00"; + deepClone = false; + leaveDotGit = false; + path = "/nix/store/b3h25asfh205wzwjfzjf2k2kkccpp96k-stdx-allocator"; + }; +} ] \ No newline at end of file diff --git a/src/build_depends/dub2nix/dub2nix.nix b/src/build_depends/dub2nix/dub2nix.nix new file mode 100644 index 0000000..ef4bd5f --- /dev/null +++ b/src/build_depends/dub2nix/dub2nix.nix @@ -0,0 +1,17 @@ +{pkgs}: +with (import ./mkDub.nix { + inherit pkgs; +}); +mkDubDerivation { + src = pkgs.lib.cleanSource ./.; + # dubJSON = ./dub.json; + # selections = ./dub.selections.nix; + version = "0.2.4"; + # doCheck = true; + propagatedBuildInputs = [ pkgs.nix-prefetch-git pkgs.cacert ]; + meta = with pkgs.stdenv.lib; { + homepage = "https://github.com/lionello/dub2nix"; + maintainers = [ maintainers.lionello ]; + license = licenses.mit; + }; +} diff --git a/src/build_depends/dub2nix/mkDub.nix b/src/build_depends/dub2nix/mkDub.nix new file mode 100644 index 0000000..7fddf14 --- /dev/null +++ b/src/build_depends/dub2nix/mkDub.nix @@ -0,0 +1,121 @@ +{ pkgs ? import {}, + stdenv ? pkgs.stdenv, + rdmd ? pkgs.rdmd, + dmd ? pkgs.dmd, + dub ? pkgs.dub }: + +with stdenv; +let + # Filter function to remove the .dub package folder from src + filterDub = name: type: let baseName = baseNameOf (toString name); in ! ( + type == "directory" && baseName == ".dub" + ); + + # Convert a GIT rev string (tag) to a simple semver version + rev-to-version = builtins.replaceStrings ["v" "refs/tags/v"] ["" ""]; + + dep2src = dubDep: pkgs.fetchgit { inherit (dubDep.fetch) url rev sha256 fetchSubmodules; }; + + # Fetch a dependency (source only for now) + fromDub = dubDep: mkDerivation rec { + name = "${src.name}-${version}"; + version = rev-to-version dubDep.fetch.rev; + nativeBuildInputs = [ rdmd dmd dub ]; + src = dep2src dubDep; + + buildPhase = '' + runHook preBuild + export HOME=$PWD + dub build -b=release + runHook postBuild + ''; + + # outputs = [ "lib" ]; + + # installPhase = '' + # runHook preInstall + # mkdir -p $out/bin + # runHook postInstall + # ''; + }; + + # Adds a local package directory (e.g. a git repository) to Dub + dub-add-local = dubDep: "dub add-local ${(fromDub dubDep).src.outPath} ${rev-to-version dubDep.fetch.rev}"; + + # The target output of the Dub package + targetOf = package: "${package.targetPath or "."}/${package.targetName or package.name}"; + + # Remove reference to build tools and library sources + disallowedReferences = deps: [ dmd rdmd dub ] ++ builtins.map dep2src deps; + + removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}''; + +in { + inherit fromDub; + + mkDubDerivation = lib.makeOverridable ({ + src, + nativeBuildInputs ? [], + dubJSON ? src + "/dub.json", + selections ? src + "/dub.selections.nix", + deps ? import selections, + passthru ? {}, + package ? lib.importJSON dubJSON, + ... + } @ attrs: stdenv.mkDerivation ((removeAttrs attrs ["package" "deps" "selections" "dubJSON"]) // { + + pname = package.name; + + nativeBuildInputs = [ rdmd dmd dub pkgs.removeReferencesTo ] ++ nativeBuildInputs; + disallowedReferences = disallowedReferences deps; + + passthru = passthru // { + inherit dub dmd rdmd pkgs; + }; + + src = lib.cleanSourceWith { + filter = filterDub; + src = lib.cleanSource src; + }; + + preFixup = '' + find $out/bin -type f -exec ${removeExpr (disallowedReferences deps)} '{}' + || true + ''; + + buildPhase = '' + runHook preBuild + + export HOME=$PWD + ${lib.concatMapStringsSep "\n" dub-add-local deps} + dub build -b release --combined --skip-registry=all + + runHook postBuild + ''; + + checkPhase = '' + runHook preCheck + + export HOME=$PWD + ${lib.concatMapStringsSep "\n" dub-add-local deps} + dub test --combined --skip-registry=all + + runHook postCheck + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + cp -r "${targetOf package}" $out/bin + + runHook postInstall + ''; + + meta = lib.optionalAttrs (package ? description) { + description = package.description; + } // attrs.meta or {}; + } // lib.optionalAttrs (!(attrs ? version)) { + # Use name from dub.json, unless pname and version are specified + name = package.name; + })); +} diff --git a/src/build_depends/dub2nix/shell.nix b/src/build_depends/dub2nix/shell.nix new file mode 100644 index 0000000..002ff55 --- /dev/null +++ b/src/build_depends/dub2nix/shell.nix @@ -0,0 +1,16 @@ +with import {}; + +let + pkg = import ./default.nix { inherit pkgs; }; + +in mkShell { + + buildInputs = [ + # additional runtime dependencies go here + ] ++ pkg.buildInputs ++ pkg.propagatedBuildInputs; + + nativeBuildInputs = [ + # additional dev dependencies go here + ] ++ pkg.nativeBuildInputs; + +} diff --git a/src/build_depends/dub2nix/src/dub2nix.d b/src/build_depends/dub2nix/src/dub2nix.d new file mode 100644 index 0000000..e92ecf8 --- /dev/null +++ b/src/build_depends/dub2nix/src/dub2nix.d @@ -0,0 +1,263 @@ +#!/usr/bin/env dub +/+ dub.sdl: + name "dub2nix" + stringImportPaths "." + dependency "vibe-d:data" version="*" ++/ +import vibe.data.json, std.string; + +enum mkDubNix = import("./mkDub.nix"); +enum VERSION = "0.2.4"; +enum HEADER = "# This file was generated by https://github.com/lionello/dub2nix v"~VERSION~"\n"; + +unittest { + static assert(import("./dub2nix.nix").indexOf(VERSION) > 0, "VERSION does not match version in dub2nix.nix"); +} + +struct DubSelections { + int fileVersion; + string[string] versions; +} + +struct DubRepo { + string owner; + string kind; + string project; +} + +private string packageRegistry = "http://code.dlang.org/packages/"; + +private auto download(string url) @trusted { +version(none) { + // This works, but causes "leaking eventcore driver" warnings at shutdown + import vibe.http.client : requestHTTP; + scope res = requestHTTP(); + return res.readJson(); +} else { + import std.net.curl: get, HTTP; + auto http = HTTP(); + // Using deflate saves A LOT of traffic, ~40x + http.addRequestHeader("accept-encoding", "deflate"); + http.addRequestHeader("accept", "application/json"); + const data = get(url, http); + // Only accepting application/json, so anything else must be compressed + if (data[0] != '{') { + import std.zlib : uncompress; + return parseJsonString(cast(string)uncompress(data)); + } else { + // parseJsonString takes immutable string, so need the .idup here + return parseJsonString(data.idup); + } +} +} + +/// Query Dub registry for the repository information of a package +auto findRepo(string pname) @safe { + const url = packageRegistry ~ pname ~ ".json"; + const json = download(url); + return deserializeJson!DubRepo(json["repository"]); +} + +struct NixPrefetchGit { + @optional string type; /// set to "git", like Go deps.nix + string url; /// URL of GIT repository + string rev; /// sha1 or tag + string sha256; /// calculated by from nix-prefetch-git + @optional bool fetchSubmodules; /// optional; defaults to true + @optional string date; /// ignored; fetchgit doesn't actually want this + @optional bool deepClone; /// ignored + @optional bool leaveDotGit; /// ignored; if the .git directory should be preserved + @optional string path; /// ignored; a path in the Nix store? +} + +/// Invoke nix-prefetch-git and return the parsed JSON +auto nixPrefetchGit(string url, string rev) @safe { + import std.process : executeShell, Config; + const cmd = "nix-prefetch-git --quiet " ~ url ~ " " ~ rev; + return deserializeJson!NixPrefetchGit( + executeShell(cmd, null, Config.stderrPassThrough).output + ); +} + +struct DubDep { + NixPrefetchGit fetch; /// like Go deps.nix +} + +/// Fetch the repo information for package `pname` and version `ver` +auto prefetch(string pname, string ver) @safe { + const repo = findRepo(pname); + assert(repo.kind == "github"); + const url = "https://" ~ repo.kind ~ ".com/" ~ repo.owner ~ '/' ~ repo.project ~ ".git"; + const tag = "v" ~ ver; + auto set = nixPrefetchGit(url, tag); + // Overwrite the sha1 ref with the tag instead, so we have the version info as well + set.rev = tag; + set.type = "git"; + return DubDep(set); +} + +/// Convert D string to Nix string literal +auto toNixString(in string s, int indent = 0) pure @safe { + if (s is null) { + return "null"; + } else if (s.indexOfAny("\"\n") >= 0) + return "''\n" ~ s ~ "''"; + else + return '"' ~ s ~ '"'; +} + +unittest { + static assert(toNixString(null) == "null"); + static assert(toNixString("hello") == `"hello"`); + static assert(toNixString("with\nnewline") == "''\nwith\nnewline''"); + static assert(toNixString(`with "quotes"`) == "''\nwith \"quotes\"''"); +} + +/// Convert D bool to Nix boolean literal +auto toNixString(bool b, int indent = 0) pure @safe { + return b ? "true" : "false"; +} + +unittest { + static assert(toNixString(true) == "true"); + static assert(toNixString(false) == "false"); +} + +private enum INDENT = " "; + +/// Convert D struct to Nix set +auto toNixString(T)(in T pod, int indent = 0) pure @safe if (is(T == struct)) { + string prefix = INDENT[0..indent * 2 + 2]; + string set = "{\n"; + foreach(i, ref key; pod.tupleof) { + const id = __traits(identifier, pod.tupleof[i]); + set ~= prefix ~ id ~ " = " ~ toNixString(key, indent + 1) ~ ";\n"; + } + return set ~ INDENT[0..indent * 2] ~ "}"; +} + +unittest { + struct TestStruct { bool b; } + static assert(toNixString(TestStruct.init) == "{\n b = false;\n}"); + static assert(toNixString(TestStruct.init, 1) == "{\n b = false;\n }"); +} + +/// Convert D AArray to Nix set +auto toNixString(T)(in T[string] aa, int indent = 0) pure @safe { + string prefix = INDENT[0..indent * 2 + 2]; + string set = "{\n"; + foreach(id, ref key; aa) { + set ~= prefix ~ id ~ " = " ~ toNixString(key, indent + 1) ~ ";\n"; + } + return set ~ INDENT[0..indent * 2] ~ "}"; +} + +unittest { + static assert(toNixString(["s": "x"]) == "{\n s = \"x\";\n}"); + static assert(toNixString(["s": ["x": true]]) == "{\n s = {\n x = true;\n };\n}"); +} + +/// Convert D array/range to Nix list +import std.range : isForwardRange; +auto toNixString(R)(in R range, int indent = 0) pure @safe if (isForwardRange!R && !is(R : string)) { + string list = "[ "; + foreach(const ref item; range) { + list ~= toNixString(item, indent) ~ " "; + } + return list ~ "]"; +} + +unittest { + static assert(toNixString(["a"]) == `[ "a" ]`); +} + +/// Create Nix expression for all dependencies in the selections JSON +auto createNixDeps(string selectionsJson) { + import std.parallelism : taskPool; + import std.array : byPair, array; + import std.stdio : writeln; + + const selections = deserializeJson!DubSelections(selectionsJson); + assert(selections.fileVersion == 1); + + static auto progress(Tuple)(in Tuple pair) { + writeln("# Prefetching ", pair.key, "-", pair.value); + return prefetch(pair.key, pair.value); + } + + // Fetch all dependency information in parallel + debug scope(success) writeln("# Done."); + return HEADER ~ toNixString(taskPool.amap!progress(selections.versions.byPair.array)); +} + +unittest { + enum json = import("./dub.selections.json"); + enum fixture = import("./dub.selections.nix"); + assert(createNixDeps(json) == fixture); +} + +// No "main" when we're running with unittests +version(unittest) {} else { + +int main(string[] args) { + import std.stdio : writeln; + import std.file : readText, write; + import std.getopt: getopt, defaultGetoptPrinter; + + bool showVersion; + string input = "./dub.selections.json", deps = "./dub.selections.nix", output; + auto result = getopt(args, + "input|i|in", "Path of selections JSON; defaults to " ~ input, &input, + "output|o|out", "Output Nix file for Dub project.", &output, + "registry|r", "URL to Dub package registry; default " ~ packageRegistry, &packageRegistry, + "deps-file|d", "Output Nix file with dependencies; defaults to " ~ deps, &deps, + "version", "Show version information.", &showVersion); + + if (showVersion) { + writeln(VERSION); + return 0; + } else if (result.helpWanted || args.length != 2 || args[1] != "save") { + defaultGetoptPrinter(`Usage: dub2nix [OPTIONS] COMMAND + +Create Nix derivations for Dub package dependencies. + +Commands: + save Write Nix files for Dub project + +Options:`, result.options); + return 0; + } + + try { + const nix = createNixDeps(readText(input)); + if (deps == "-") { + writeln(nix); + } else { + write(deps, nix.representation); + } + if (output) { + write("mkDub.nix", mkDubNix); + write(output, HEADER ~ ` +{ pkgs ? import {} }: +with import ./mkDub.nix { inherit pkgs; }; + +mkDubDerivation { + src = ./.; + # version = "0.0.1"; + # buildInputs = [ add any runtime deps here ]; +} +`); + } + return 0; + } catch (Exception e) { + debug { + // Only dump callstack in debug builds + writeln(e.toString()); + } else { + writeln("Error: ", e.message); + } + return 1; + } +} + +}//!unittest -- cgit v1.2.3