From 67b5c45076a3744c2c4879a0ef017c0751611ec2 Mon Sep 17 00:00:00 2001 From: Jeremy Wall Date: Sun, 3 Apr 2022 17:19:40 -0400 Subject: [PATCH] Convert over to flake. --- .gitignore | 3 +- dbt-core/default.nix | 169 ----------------------------------- flake.lock | 78 +++++++++++++++++ flake.nix | 27 ++++++ nix/dbt-core/default.nix | 82 +++++++++++++++++ nix/dbt-core/overlay.nix | 185 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 374 insertions(+), 170 deletions(-) delete mode 100644 dbt-core/default.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/dbt-core/default.nix create mode 100644 nix/dbt-core/overlay.nix diff --git a/.gitignore b/.gitignore index d5813d2..fcd749b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ lib/* bin/* pyvenv.cfg -share/* \ No newline at end of file +share/* +result diff --git a/dbt-core/default.nix b/dbt-core/default.nix deleted file mode 100644 index fbfeed4..0000000 --- a/dbt-core/default.nix +++ /dev/null @@ -1,169 +0,0 @@ -with import (builtins.fetchTarball { - name = "nixpkgs-21.05-darwin"; - url = "https://github.com/nixos/nixpkgs/archive/06b49ba179e3e5b13364ed16aa9907821abc6988.tar.gz"; - sha256 = "18dij8g8p71a3ymr58bjn9j7bl9d3hkmzfccc0bqk5fi887i4z7z"; -}) { - overlays = [ - # Overlay our packages to the correct versions. - # TODO my naming here is wrong and therefore confusing - (self: super: { - python39 = super.python39.override { - packageOverrides = (pyself: pysuper: { - sqlparse = pysuper.sqlparse.overrideAttrs (oldAttrs: rec { - version = "0.2.3"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "1y7gpqjk15ccgya032214iq2j0r4pprgrvffx0fk3pxvrv3prkdy"; - }; - }); - colorama = pysuper.colorama.overrideAttrs (oldAttrs: rec { - version = "0.3.9"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "1wd1szk0z3073ghx26ynw43gnc140ibln1safgsis6s6z3s25ss8"; - }; - }); - jsonschema = pysuper.jsonschema.overrideAttrs (oldAttrs: rec { - version = "2.6.0"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "00kf3zmpp9ya4sydffpifn0j0mzm342a2vzh82p6r0vh10cg7xbg"; - }; - }); - boto3 = pysuper.boto3.overrideAttrs (oldAttrs: rec { - version = "1.6.23"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "0r28nv8wfrqc0wgacpv939wvw1cqp99wlskwg6nzx3b29d1j6gsj"; - }; - }); - botocore = pysuper.botocore.overrideAttrs (oldAttrs: rec { - version = "1.9.23"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "147sxvx8gsh2fajxx5rm1r07zvlbhwj42dckkbs5wf3s9j21500m"; - }; - }); - python-dateutil = pysuper.python-dateutil.overrideAttrs (oldAttrs: rec { - version = "2.1"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "1vlx0lpsxjxz64pz87csx800cwfqznjyr2y7nk3vhmzhkwzyqi2c"; - }; - pythonImportsCheck = [ ]; - }); - #networkx = pysuper.networkx.overrideAttrs (oldAttrs: rec { - # version = "1.11"; - # pname = oldAttrs.pname; - # src = pyself.fetchPypi { - # inherit pname; - # inherit version; - # sha256 = "1f74s56xb4ggixiq0vxyfxsfk8p20c7a099lpcf60izv1php03hd"; - # }; - #}); - certifi = pyself.buildPythonPackage rec { - version = "2020.6.20"; - pname = "certifi"; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "1lrlxvcaab3kyr5j08dgvw5cvhpij38dldfwp0dx4va92xc5jc2r"; - }; - - pythonImportsCheck = [ "certifi" ]; - - doCheck = false; - - meta = with self.lib; { - homepage = "https://certifi.io/"; - description = "Python package for providing Mozilla's CA Bundle"; - license = licenses.isc; - }; - }; - snowflake-connector-python = pysuper.snowflake-connector-python.overrideAttrs (oldAttrs: rec { - version = "2.4.1"; - pname = oldAttrs.pname; - src = pyself.fetchPypi { - inherit pname; - inherit version; - sha256 = "1bms4z3zjxzzg0m9smgf0h5cm49h8a41c8w3vyqvx9q22bk814aw"; - }; - }); - }); - }; - }) - ]; -}; -with python39Packages; - -buildPythonPackage rec { - pname = "dbt"; - version = "0.12.0"; - - propagatedBuildInputs = [ - attrs - Babel - agate - certifi - cffi - click - colorama - hologram - idna - importlib-metadata - isodate - jinja2 - jsonschema - leather - Logbook - markupsafe - msgpack - networkx - packaging - parsedatetime - psycopg2 - pycparser - pyparsing - pyrsistent - python-dateutil - python-slugify - pytimeparse - pytz - pyyaml - requests - six - sqlparse - text-unidecode - typing-extensions - urllib3 - werkzeug - zipp - snowflake-connector-python - #charset-normalizer - #mashumaro - #minimal-snowplow-tracker - #dbt-extractor - #dbt-postgres - ]; - - doCheck = false; - - src = fetchPypi { - inherit pname; - inherit version; - #sha256 = "0000000000000000000000000000000000000000000000000000"; - sha256 = "0ymd6pp0vw32s9y3dwcb083ds1cz8y6qiiwcajcyj07ld1wxmc0x"; - }; -} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..51454ab --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1648297722, + "narHash": "sha256-W+qlPsiZd8F3XkzXOzAoR+mpFqzm3ekQkJNa+PIh1BQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "0f8662f1319ad6abf89b3380dd2722369fc51ade", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "locked": { + "lastModified": 1637014545, + "narHash": "sha256-26IZAc5yzlD9FlDT54io1oqG/bBoyka+FJk5guaX4x4=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "bba5dcc8e0b20ab664967ad83d24d64cb64ec4f4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1648219316, + "narHash": "sha256-Ctij+dOi0ZZIfX5eMhgwugfvB+WZSrvVNAyAuANOsnQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "30d3d79b7d3607d56546dd2a6b49e156ba0ec634", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1648953213, + "narHash": "sha256-kXWcXFwqcvooHjcDoEK4mtvKU/8LuetwhrofU+9KMS0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "caa1a7ea867138d4f4e05bd2f41d707df0a61cde", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..a0e1194 --- /dev/null +++ b/flake.nix @@ -0,0 +1,27 @@ +{ + inputs = { + nixpkgs.url = "nixpkgs"; + flake-utils.url = "github:numtide/flake-utils"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = {self, nixpkgs, rust-overlay, flake-utils, ...}: + flake-utils.lib.eachDefaultSystem(system: + let + dbt-overlay = import ./nix/dbt-core/overlay.nix; + # TODO(jwall): Is this overlay strictly necessary? + pkgs = import nixpkgs {inherit system; overlays = [ dbt-overlay rust-overlay.overlay ]; }; + dbt-core = import ./nix/dbt-core/default.nix { inherit pkgs; }; + in + { + overlays = [ dbt-overlay rust-overlay ]; + defaultPackage = dbt-core; + packages.dbt-core = dbt-core; + devShell = pkgs.mkShell { + buildInputs = [ dbt-core ]; + }; + }); +} \ No newline at end of file diff --git a/nix/dbt-core/default.nix b/nix/dbt-core/default.nix new file mode 100644 index 0000000..b8b6b10 --- /dev/null +++ b/nix/dbt-core/default.nix @@ -0,0 +1,82 @@ +{pkgs}: +with pkgs; +with python39Packages; +let inputs = [ + #attrs + #Babel + #boto3 + #agate + #certifi + #cffi + #click + #colorama + #hologram + #idna + #importlib-metadata + #isodate + #jinja2 + #leather + #Logbook + #markupsafe + #msgpack + #networkx + #packaging + #parsedatetime + #psycopg2 + #pycparser + #pyparsing + #pyrsistent + #python-dateutil + #python-slugify + #pytimeparse + #pytz + #pyyaml + #requests + #sqlparse + #text-unidecode + #typing-extensions + #urllib3 + #werkzeug + #zipp + #pyopenssl + #vcversioner + #six + #snowflake-connector-python + #charset-normalizer + #dbt-postgres + jsonschema + networkx + mashumaro + hologram + colorama + packaging + requests + agate + minimal-snowplow-tracker + click + werkzeug + jinja2 + logbook + dbt-extractor + typing-extensions + sqlparse + python-dateutil + msgpack + ]; +in +buildPythonPackage rec { + pname = "dbt-core"; + version = "1.0.0"; + + buildInputs = inputs; + + propagatedBuildInputs = inputs; + + doCheck = false; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-kL3gHQeLHWc9xJC53v/wfn87cCuSDGX+rm+yfgfOc9I="; + }; + +} diff --git a/nix/dbt-core/overlay.nix b/nix/dbt-core/overlay.nix new file mode 100644 index 0000000..1ee17b2 --- /dev/null +++ b/nix/dbt-core/overlay.nix @@ -0,0 +1,185 @@ +(self: super: { + python39 = super.python39.override { + packageOverrides = (pyself: pysuper: rec { + dbt-extractor = pyself.buildPythonPackage rec { + pname = "dbt_extractor"; + version = "0.4.0"; + format = "pyproject"; + nativeBuildInputs = with self.rustPlatform; [ + cargoSetupHook maturinBuildHook + ]; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-WGcuNvq5iMhJppNAWSDuGEIfJyRcSOX57PSWNp7TGoU="; + }; + cargoDeps = self.rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-FXMIatCTZJSLaABYO/lcBsONXw8FPNQmcm/tRwkSE54="; + patchPhase = '' + pwd + ls + # You can't use the unauthenticated git url format anymore + sed -i.bak '/tree-sitter-jinja2/ { s/git:/https:/ }' Cargo.toml + ''; + }; + }; + hologram = pyself.buildPythonPackage rec { + pname = "hologram"; + version = "0.0.14"; + buildInputs = with pyself; [ + jsonschema + python-dateutil + colorama + ]; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-/We9Bp5GgeHSpEffl2xlBg16kP7n9rhNEz/ZlY2wdOw="; + }; + # The tests use the network so disable for now. + doCheck = false; + }; + colorama = pysuper.colorama.overrideAttrs (oldAttrs: rec { + version = "0.3.9"; + pname = oldAttrs.pname; + src = pyself.fetchPypi { + inherit pname; + inherit version; + sha256 = "1wd1szk0z3073ghx26ynw43gnc140ibln1safgsis6s6z3s25ss8"; + }; + }); + networkx = pyself.buildPythonPackage rec { + pname = "networkx"; + version = "2.7.1"; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-0RlLp1Pl7tB83s0dI8XNejx3IJm9jb0v6jZniM9N57o="; + }; + # The tests use the network so disable for now. + doCheck = false; + }; + mashumaro = pyself.buildPythonPackage rec { + pname = "mashumaro"; + version = "2.9"; + buildInputs = [ + msgpack + typing-extensions + pyself.pyyaml + networkx + ]; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-NDtuLT5DLjGXNojEyIIdzW70H9MyZLmSr8Suy/0VXxg="; + }; + }; + jsonschema = pyself.buildPythonPackage rec { + pname = "jsonschema"; + version = "3.0.0"; + + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-rMipDDHREGBRbP0LQUufi89LxpGyHw94bqV91SVceds="; + }; + + nativeBuildInputs = with pyself; [ setuptools-scm ]; + propagatedBuildInputs = with pyself; [ setuptools attrs importlib-metadata functools32 pyrsistent ]; + checkInputs = with pyself; [ nose mock pyperf twisted vcversioner ]; + + # zope namespace collides on py27 + doCheck = false; + checkPhase = '' + nosetests + ''; + }; + minimal-snowplow-tracker = with pyself; buildPythonPackage rec { + pname = "minimal-snowplow-tracker"; + version = "0.0.2"; + buildInputs = [ six requests ]; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-rKv3Vy2w5/XL9pg9SV7vVAgfcb45IzDrOq25zLOdqqQ="; + }; + }; + jinja2 = pysuper.jinja2.overrideAttrs (oldAttrs: rec { + version = "2.11.3"; + pname = oldAttrs.pname; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-ptWEM94K6AA0fKsfowQ867q+i6qdKeZo8cdoy4ejM8Y="; + }; + }); + logbook = with pyself; buildPythonPackage rec { + pname = "Logbook"; + version = "1.5.3"; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-ZvRUraD1bq5DBm9gSiIrCYk/mMGtwY3xaXEHYbjzL+g="; + }; + # The tests use the network so disable for now. + doCheck = false; + }; + typing-extensions = with pyself; buildPythonPackage rec { + pname = "typing_extensions"; + version = "3.7.4.3"; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-mdQHO2F9MCiPVp0/E9K9dUjDp+TI3ofbCanSm7Okpgw="; + }; + }; + python-dateutil = pysuper.python-dateutil.overrideAttrs (oldAttrs: rec { + pname = oldAttrs.pname; + version = "2.8.0"; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-yJgF9vTWTbIe2Wb9oTj4pe16T9vBqO4ynOG3Tjx02p4="; + }; + }); + msgpack = pysuper.msgpack.overrideAttrs (oldAttrs: rec { + pname = oldAttrs.pname; + version = "0.5.6"; + src = pyself.fetchPypi { + inherit pname version; + sha256 = "sha256-DujIyFqmUb46oM0AW1kxdp6qZYyUjOeUKHZvG9Rq4sM="; + }; + doCheck = false; + installCheckPhase = ""; + }); + #certifi = pyself.buildPythonPackage rec { + # version = "2020.6.20"; + # pname = "certifi"; + # src = pyself.fetchPypi { + # inherit pname; + # inherit version; + # sha256 = "1lrlxvcaab3kyr5j08dgvw5cvhpij38dldfwp0dx4va92xc5jc2r"; + # }; + + # pythonImportsCheck = [ "certifi" ]; + # + # doCheck = false; + + # meta = with self.lib; { + # homepage = "https://certifi.io/"; + # description = "Python package for providing Mozilla's CA Bundle"; + # license = licenses.isc; + # }; + #}; + #vcversioner = pyself.buildPythonPackage rec { + # version = "2.16.0.0"; + # pname = "vcversioner"; + # src = pyself.fetchPypi { + # inherit pname version; + # sha256 = "sha256-2uYMF6R5eB9EpAEHAYM/GCkUCx7szSWHYqdJdKoG4Zs="; + # }; + #}; + #snowflake-connector-python = pysuper.snowflake-connector-python.overrideAttrs (oldAttrs: rec { + # version = "2.4.1"; + # pname = oldAttrs.pname; + # src = pyself.fetchPypi { + # inherit pname; + # inherit version; + # sha256 = "1bms4z3zjxzzg0m9smgf0h5cm49h8a41c8w3vyqvx9q22bk814aw"; + # }; + #}); + }); + }; + }) \ No newline at end of file