Feat/refine template (#13)

* feat: add dev mode

* fix: proxy

* feat: make secops nicer

* prepare basics for components

* k

* feat: add components

* k

* k

* k

* k

* k

* k

* k

* k

* kk

* k

* k

* k

* k

* k

* k

* process_dbt_assets

* k

* k

* k

* k

* k

* k

Author: geoHeil

.github/workflows/validate-template.yaml

@@ -0,0 +1,52 @@
+name: Validation of template
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - 'pyproject.toml'
+      - 'pixi.lock'
+      - 'projects/**'
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: [ main ]
+    paths:
+      - 'pyproject.toml'
+      - 'pixi.lock'
+      - 'projects/**'
+  workflow_dispatch:
+
+concurrency:
+  group: tutorial-main-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  validation:
+    name: Validate tutorial
+    runs-on: ubuntu-latest
+    env:
+      # dagster dbt needs to run in prepare_if_dev to parse the project
+      DAGSTER_IS_DEV_CLI: "True"
+      CATALOG_DB_USER: "dummy"
+      CATALOG_DB_PASSWORD: "dummy"
+      OBJECT_STORE_ROOT_USER: "dummy"
+      OBJECT_STORE_ROOT_PASSWORD: "dummy"
+      OBJECT_STORE_REGION: "dummy"
+      OPENAI_API_KEY: "dummy"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: prefix-dev/setup-pixi@v0.9.0
+        with:
+          pixi-version: v0.55.0
+          cache: true
+          frozen: true
+      
+     # Create an instance of the template
+      - run: pixi run render-dev
+
+     # check the instance can start
+      - run: cd rendered-template/local_data_stack/ && pixi run -e dev test
+
+    # possibly add further testing later

.gitignore

@@ -1,3 +1,4 @@
 .pixi/envs/
 .pixi/solve-group-envs/
 local_data_stack
+key.txt

README.md

@@ -30,6 +30,8 @@ Additionally included for secure handling of secrets:
 - sops
 - age
 
+![](img/ducking-the-lake.png)
+
 ## usage
 
 Prerequisites: 
@@ -74,4 +76,11 @@ pixi run start-dev
 Post install:
 
 - update the secrets in the `.env` files by executing: `openssl rand -base64 32` and setting a suitable secret
-- ensure the `.env.enc` can be created by following the instructions in [documentation/secops]
+- ensure the `.env.enc` can be created by following the instructions in [documentation/secops]
+
+## Developing the template
+
+To render an instance of the project:
+```bash
+pixi run render-dev
+```

img/ducking-the-lake.png

@@ -37,3 +37,10 @@ env = { RUST_LOG = "warn" }
 cmd = "pixi run -e template cruft create https://github.com/l-mds/local-data-stack.git"
 description = "Initialize template with cruft"
 
+[tool.pixi.tasks.render-dev]
+cmd = "pixi run --frozen -e template cruft create . --no-input --overwrite-if-exists --output-dir rendered-template && ./scripts/handle-dev-secrets.sh"
+description = "locally render a development instance of the template"
+
+[tool.pixi.tasks.render-dev-no-secrets]
+cmd = "pixi run --frozen -e template cruft create . --no-input --overwrite-if-exists --output-dir rendered-template"
+description = "locally render a development instance of the template; no secrets needed (for CI)"

pixi.lock

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cp key.txt rendered-template/local_data_stack/
+cd rendered-template/local_data_stack
+make secrets-decrypt
+cd ../../

pyproject.toml

@@ -1,7 +1,5 @@
-# syntax=docker/dockerfile:1.10.0
-
 # for CUDA support ensure docker is nvidia-docker enabled!
-FROM ghcr.io/prefix-dev/pixi:0.41.1-bookworm AS builder_base
+FROM ghcr.io/prefix-dev/pixi:0.55.0-bookworm AS builder_base
 
 FROM builder_base AS build
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/

scripts/handle-dev-secrets.sh

@@ -35,13 +35,13 @@ fmt-unsafe:
 .PHONY: lint
 ## Ruff based flake8 style linting plus type checking via pyright
 lint:
-	pixi run -e ci-validation lint
+	pixi run -e dev lint
 
 
 .PHONY: test
 ## Execute tests with coverage
 test:
-	pixi run -e ci-validation test
+	pixi run -e dev test
 
 
 

{{ cookiecutter.project_slug }}/Dockerfile

@@ -12,7 +12,7 @@ pixi update
 
 
 # launch a shell
-pixi shell --frozen -e ci-validation
+pixi shell --frozen -e dev
 cd src/code_location_{{ cookiecutter.project_slug }}/code_location_{{ cookiecutter.project_slug }}_dbt/ && dbt deps
 cd ../../../
 ```
@@ -23,11 +23,11 @@ set `DAGSTER_HOME` to have persistent logs
 
 ```bash
 # for a single code location
-pixi shell --frozen -e ci-validation
+pixi shell --frozen -e dev
 
 dagster dev
 # alternatively:
-pixi run -e ci-validation start-dev
+pixi run -e dev start-dev
 
 
 dagster job list --location foo