diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..350c0a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,37 @@ +--- +name: Bug Report +about: Create a report to help us improve ZCore +title: 'bug: ' +labels: bug +assignees: '' +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Steps to reproduce: +2. Code snippet or minimal reproducible example (highly recommended): +```python +# Paste your minimal code here +``` +3. Run command '...' +4. See error/traceback. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Traceback / Logs** +If applicable, paste your structured logs or console tracebacks here: +```text +# Paste raw logs or tracebacks here +``` + +**Environment (please complete the following information):** +- Python Version: [e.g. 3.11] +- FastAPI Version: [e.g. 0.110.0] +- ZCore Version: [e.g. 0.1.0-beta.5] +- SQLAlchemy Version: [e.g. 2.0.28] (if applicable) +- Database Engine: [e.g. SQLite (aiosqlite) / PostgreSQL (asyncpg)] +- Redis Version: [e.g. 5.0.3] (if using cache or streams) \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..b656b51 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature Request +about: Suggest an idea or enhancement for ZCore +title: 'feat: ' +labels: enhancement +assignees: '' +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is (e.g. "I'm frustrated when trying to run...") + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. How should the API look or behave under ZCore? + +**How does this align with ZCore's Design Principles?** +Please explain how this feature complements FastAPI without hiding its core features, and how it fits into ZCore's modular philosophy. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context, code blocks, or references about the feature request here. \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7969672 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,30 @@ +## Description + +Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. + +Fixes # (issue number) + +## Type of change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Refactoring (architectural or code cleanup with no functional changes) +- [ ] Documentation update + +## How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. + +- [ ] Unit/Integration tests (`hatch run test`) +- [ ] Static type check passing (`mypy src/`) + +## Checklist: + +- [ ] My code follows the style guidelines and architectural rules of this project +- [ ] I have performed a self-review of my own code +- [ ] My changes pass strict static type analysis (`mypy src/` generates no warnings) +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation (under `docs/`) +- [ ] My changes generate no new runtime warnings +- [ ] I have added tests that prove my fix is effective or that my feature works \ No newline at end of file diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..d651724 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,29 @@ +changelog: + exclude: + labels: + - ignore + authors: + - dependabot[bot] + categories: + - title: 'Features πŸš€' + labels: + - enhancement + - feat + - backend + - title: 'Bug Fixes πŸ›' + labels: + - bug + - fix + - title: 'Refactoring & Architecture πŸ› οΈ' + labels: + - refactor + - security + - di + - database + - title: 'Documentation πŸ“' + labels: + - documentation + - docs + - title: 'Dependencies & Maintenance βš™οΈ' + labels: + - dependencies \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3ba36bd --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,35 @@ +name: Test Suite (CI) + +on: + pull_request: + branches: + - master + - develop + push: + branches: + - master + - develop + +jobs: + test: + name: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Upgrade pip + run: python -m pip install --upgrade pip + + - name: Install Hatch and Ruff + run: pip install hatch ruff + + - name: Run Ruff Lint Check + run: ruff check src/zcore + + - name: Run Hatch Tests + run: hatch test \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bf991cd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,167 @@ +# Contributing to ZCore + +First, thank you for considering contributing to ZCore! It is through developers like you that we can continue to refine this architectural chassis for FastAPI. + +Please take a moment to review this document to ensure your contributions align with our design philosophy, codebase standards, and development workflows. + +--- + +## πŸ›οΈ Our Design Principles + +When proposing changes or writing code for ZCore, please keep these core pillars in mind: +1. **The Complementary Promise:** ZCore must never obscure or lock developers out of native FastAPI/Pydantic capabilities. Your contributions should enhance, not restrict, developer freedom. +2. **Strict Typings:** ZCore is designed with highly strict type safety. All public APIs must be fully annotated and pass static analysis checks without warnings. +3. **Fail-Safe Security:** Security defaults (like magic byte verification or secure fallback JWT checks) should be enforced out-of-the-box, protecting developers from accidental vulnerabilities. +4. **Performance Pragmatism:** Performance over-optimization is a trap, but unnecessary overhead (like repetitive reflection without caching) must be avoided. + +### β›” What We Don't Accept +To preserve ZCore's lightweight, explicit, and highly predictable nature, we generally avoid: +- **Tight Coupling:** Features that tightly bind ZCore to FastAPI internals or prevent independent unit testing of core components. +- **Implicit "Magic" Behavior:** We prefer explicit, clear configuration over automatic, hard-to-debug "magic" conventions. +- **Breaking Changes:** Any breaking changes submitted without a clearly documented and seamless migration path. +- **Heavy Runtime Overhead:** Third-party dependencies or features that introduce unnecessary CPU/memory overhead (e.g., repetitive reflection without proper caching layers). + +--- + +## 🚦 Before You Contribute + +To help us maintain a high-quality codebase and manage our maintainer overhead, please follow these steps before opening an issue or a pull request: + +1. **Search Existing Issues:** Ensure the bug or feature request has not already been reported or discussed. +2. **Provide a Minimal Reproducible Example (MRE):** When reporting bugs, please provide a self-contained, copy-pasteable script that reproduces the issue. Issues without a reproduction case may be closed. +3. **Include Environment Information:** Always specify your Python, FastAPI, and ZCore versions, as well as your operating system. + +--- + +## πŸ› οΈ Local Development Setup + +ZCore utilizes **Hatch** as its build backend and project manager. Follow these steps to prepare your local development environment: + +### 1. Fork and Clone +Fork the repository on GitHub, then clone your fork locally: +```bash +git clone https://github.com/your-username/zcore.git +cd zcore +``` + +### 2. Install Development Dependencies +It is highly recommended to use a virtual environment. Install ZCore in editable mode along with all optional and development dependencies: +```bash +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install with editable mode and all dev extras +pip install -e ".[all,dev]" +``` + +--- + +## πŸ§ͺ Testing Guidelines + +We aim for high test coverage on core architectural units. ZCore uses `pytest` and `pytest-asyncio` for asynchronous test suites. + +### Test Architecture +ZCore's codebase is highly modular. To ensure that components like the Kernel, Plugins, Dependency Injection (DI), Unit of Work (UOW), Router, and Middleware interact correctly, we organize our tests as follows: + +```text +tests/ +β”œβ”€β”€ unit/ # Tests for isolated, single architectural units with mocks +β”‚ β”œβ”€β”€ test_repository.py +β”‚ └── test_zchema.py +β”œβ”€β”€ integration/ # Tests verifying interactions between modules +β”‚ β”œβ”€β”€ test_kernel_startup.py +β”‚ β”œβ”€β”€ test_plugin_loading.py +β”‚ └── test_request_scope.py +└── e2e/ # Full end-to-end request/response flows using TestClient + └── test_crud_flow.py +``` + +### Running Tests Locally +Run the standard pytest suite using: +```bash +pytest +``` + +If you prefer to run tests inside Hatch-isolated environments: +```bash +hatch run test +``` + +### Writing Tests +- **Asynchronous Tests:** Ensure your async tests are decorated with `@pytest.mark.asyncio`. +- **Isolation:** Tests should not leave behind lingering SQLite databases, active sessions, or running background threads. +- **Mocks:** When writing tests that require external services like Redis, implement mock fallbacks where appropriate to ensure the test suite can run fully offline. + +--- + +## πŸ” Code Quality & Static Analysis + +Before submitting a Pull Request, your code must satisfy strict formatting, linting, and type-checking constraints. + +### 1. Type Checking +ZCore enforces strict type checks via `mypy`. To verify your changes conform to the strict rules defined in `pyproject.toml`, run: +```bash +mypy src/ +``` + +### 2. Linting & Formatting +Ensure your code is clean, PEP-8 compliant, and structured consistently. We recommend using `ruff` for formatting and linting: +```bash +ruff check src/ +ruff format src/ +``` + +--- + +## πŸ“ Commit Message Conventions + +ZCore uses the **Conventional Commits** specification to automate our changelog generation and semantic versioning. Your commit messages must follow this structure: + +```text +(): +``` + +Common types include: +- `feat`: A new feature (corresponds to a minor version bump). +- `fix`: A bug fix (corresponds to a patch version bump). +- `docs`: Documentation updates. +- `refactor`: Code changes that neither fix a bug nor add a feature. +- `test`: Adding or correcting tests. +- `ci`: Changes to our CI/CD workflows or scripts. + +*Examples:* +- `feat(di): add redis plugin support` +- `fix(uow): resolve transaction rollback issue on async commit` +- `docs(zchema): update guide for dynamic schema pruning` + +--- + +## πŸš€ Pull Request Process + +We appreciate atomic, focused Pull Requests. To increase the chances of your PR being merged quickly, please follow this process: + +```mermaid +flowchart LR + A[Create Feature Branch] --> B[Write Code & Tests] + B --> C[Pass Mypy & Pytest] + C --> D[Update Docs if needed] + D --> E[Submit PR] +``` + +1. **Branch Naming:** Use descriptive branch names (e.g., `feature/add-s3-provider` or `bugfix/fix-uow-flush`). +2. **Atomic Commits:** Keep your commits focused on single logical changes and adhere to the conventional commits schema. +3. **Documentation:** If you are adding a new feature or modifying an existing protocol, you **must** update the corresponding documentation page under the `docs/` folder. + +### πŸ›‘οΈ CI/CD & Branch Protection +Our repository has strict branch protection rules configured on `master` and `develop`. +- **Mandatory CI Checks:** Your Pull Request cannot be merged unless all status checks (Linting with Ruff, Type-checking with Mypy, and Test Suites with Pytest) pass successfully in our GitHub Actions pipeline. +- **No Direct Pushes:** All code changes must go through a Pull Request. Direct pushes to protected branches are blocked. + +--- + +## πŸ“„ Licensing & Copyright + +By contributing to ZCore, you agree that your contributions will be licensed under the project's **Apache License 2.0**. + +- **No CLA Required:** We do not currently require a signed Contributor License Agreement (CLA) for individual contributions. +- **Certificate of Origin:** By submitting a Pull Request, you assert that you have the right to submit this code and that it conforms to an open-source contribution under the Apache 2.0 terms. diff --git a/README.md b/README.md index c465ccc..f08b5a1 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,244 @@

- ZCore Logo -
- A modest and practical architectural layer built on top of FastAPI. + ZCore Logo

+ A pragmatic and complementary architectural layer built on top of FastAPI.
+ Standardize your structure, protect your data, and manage atomic transactionsβ€”without losing your development freedom. +

+ +

+ + PyPI + - License - PyPI - Build Status + License + + + CI + + + Documentation + + + Python Versions +

--- ## What is ZCore? -ZCore is not a replacement for FastAPI; it is a modest and practical architectural layer built on top of it. While FastAPI provides the high-performance engine for handling HTTP requests, ZCore provides the "chassis"β€”a structured environment that solves common challenges in medium-to-large scale applications such as dependency management, transaction integrity, and data leakage prevention. - -The framework focuses on **Engineered Simplicity**. It abstracts complex patterns like the *Unit of Work* and *Scoped Inversion of Control* into intuitive interfaces, allowing you to focus on your domain logic while the framework ensures that your database transactions are atomic and your sensitive data remains restricted based on the execution context. - ---- - -## Why Choose ZCore? +**ZCore is not a framework that hides FastAPI β€” it is the chassis that stabilizes it.** -ZCore was designed to bridge the gap between "writing an endpoint" and "building a maintainable system." +While FastAPI gives you a high-performance engine for HTTP, it provides no blueprint for structuring medium-to-large applications. ZCore fills that gap with: -| Feature | Standard FastAPI Challenge | The ZCore Approach | -| :--- | :--- | :--- | -| **Dependency Injection** | Manual wiring and complex `Depends` chains. | Automated **Scoped IoC** with constructor injection. | -| **Data Security** | Manual filtering of Pydantic models for different users. | Context-aware **Response Pruning** via `ResponseProjector`. | -| **Transactions** | Scatterred `.commit()` calls leading to partial failures. | Centralized **Unit of Work** (UOW) for atomic operations. | -| **Project Structure** | Inconsistent layouts across different teams. | Modular **Plugin System** and standardized CLI scaffolding. | -| **Search & Filter** | Writing repetitive boilerplate for every query. | A secure, dynamic **Search Engine** with depth-limit protection. | +- **πŸ” Context-aware data masking** β€” Write one schema; sensitive fields are pruned automatically per-user. +- **πŸ”— Atomic Unit of Work** β€” Group operations into all-or-nothing transactions with delayed event dispatch. +- **⚑ Scoped Dependency Injection** β€” Singleton, transient, and request-scoped dependencies resolved automatically. +- **πŸ—οΈ Modular Plugin System** β€” Organize domains into isolated plugins with topological startup ordering. +- **πŸ” Secure Dynamic Search Engine** β€” Nested JSON filters, cursor/offset pagination, and column-level security β€” zero boilerplate. +- **πŸ“¦ CLI Scaffolding** β€” From `pip install` to running CRUD endpoints in under two minutes. --- -## The Request Lifecycle +## Why ZCore? -Understanding how a request travels through ZCore is key to mastering its architecture. The following diagram illustrates the automated orchestration from the moment a request hits the server to the final pruned response. - -

- ZCore Request Lifecycle -

+| Concern | Raw FastAPI | With ZCore | +|---------|-------------|------------| +| **Endpoint Scaffolding** | Manually write 7+ files per domain | One `BaseRouter` class β†’ 7 secure endpoints | +| **Data Leakage** | Multiple Pydantic models per role; manual conditionals | `Zchema` auto-prunes restricted fields per-user | +| **Database Transactions** | Scattered `commit()` / `rollback()` calls | `UnitOfWork` guarantees atomicity + deferred events | +| **Dependency Wiring** | Deeply nested `Depends()` chains | Constructor injection with `Inject()` β€” auto-resolved | +| **Search & Pagination** | Hand-written SQL per endpoint | Declarative JSON filters + cursor/offset pagination | +| **Project Layout** | No standard; every team reinvents the wheel | `zc init` + `zc startapp` β€” consistent domain modules | +| **Startup Orchestration** | Manual `@app.on_event` spaghetti | `Plugin` with dependency graph β†’ topological sorting | --- ## ⚑ Quick Start -### 1. Installation -Set up a clean virtual environment and install ZCore with all optional dependencies: +### 1. Install ```bash -python -m venv .venv -source .venv/bin/activate # On Windows: .venv\Scripts\activate pip install fastapi-zcore-framework[all] ``` -### 2. Scaffold a Project -Initialize your project and generate a structured domain module using our command-line utility: +### 2. Scaffold ```bash -# Initialize project workspace -zc init product_api -cd product_api +zc init my_app && cd my_app +zc startapp tasks --template +``` + +### 3. Define + +Open `app/tasks/models.py`: -# Scaffold a domain app with boilerplate templates -zc startapp products -t +```python +from zcore import Base +from sqlalchemy.orm import Mapped, mapped_column + +class Task(Base): + __tablename__ = "tasks" + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] + is_completed: Mapped[bool] = mapped_column(default=False) ``` -### 3. Run the Development Server -Launch the local Uvicorn development server: +### 4. Serve ```bash zc run ``` +Your API is live at **`http://127.0.0.1:8000`** with 7 CRUD + search endpoints ready. + +> πŸ“– **Full walkthrough:** [10-Minute Quickstart](https://fastapi-zcore-framework.github.io/zcore/learn/quickstart/) + --- -## πŸ“š Core Pillars at a Glance +## πŸ›οΈ Core Pillars + +
+πŸ” Context Shielding (Zchema) +
+ +Write a single schema. ZCore dynamically prunes input fields (preventing mass assignment) and output fields (preventing data leakage) based on the active user's permission scopes. + +```python +from zcore import Zchema -* **⚑ Scoped IoC Container:** Manage object lifecycles (Singleton, Transient, or Scoped) with ease. Scoped dependencies are automatically cleared at the end of every HTTP request to prevent memory pollution. -* **πŸ›‘οΈ Secure Search Engine:** A dynamic query builder that supports nested filters and eager-loading, while automatically blocking access to restricted database columns based on security policies. -* **πŸ”— Unit of Work (UOW):** Ensures that business operations succeed or fail as a single unit. It coordinates database flushes and delays event dispatching until the transaction is successfully committed. -* **πŸ—οΈ Modular Plugin System:** Organize your application into decoupled domains. Each plugin manages its own lifecycle hooks (`on_startup`, `on_shutdown`) and can declare dependencies on other plugins. +class TaskCreate(Zchema): + __model__ = "tasks" + title: str + assignee_email: str | None = None # Auto-hidden from unauthorized users +``` + +When a user lacks `tasks:view_sensitive`, the `assignee_email` field vanishes from both validation and serialization β€” no `if` statements. +
+ +
+πŸ”— Atomic Transactions (Unit of Work) +
+ +Group multiple repository operations into a single atomic unit. Events are queued and dispatched only after a successful commit. If any operation fails, the entire transaction rolls back. + +```python +from zcore import UnitOfWork + +async with UnitOfWork(session, dispatcher) as uow: + await order_repo.create(order_data) + await inventory_repo.decrement_stock(product_id, quantity) + + # Queue event for post-commit dispatch safely + uow.register_event("order.placed", {"id": order_data.id}) +``` +
+ +
+⚑ Scoped Dependency Injection +
+ +Inject services, repositories, and infrastructure dependencies via constructor injection. ZCore's container resolves them automatically per-request and clears request-scoped instances after each response. + +```python +from zcore import BaseService, Inject + +class TaskService(BaseService): + def __init__(self, repo: TaskRepository = Inject(TaskRepository)): + super().__init__(model=Task, repository=repo) +``` +
+ +
+πŸ—οΈ Modular Plugin System +
+ +Each domain is a self-contained `Plugin` with its own models, services, routers, and lifecycle hooks. The Kernel uses topological sorting to guarantee correct startup/shutdown order. + +```python +from zcore import Plugin +from .routers import router_instance + +class TaskPlugin(Plugin): + name = "tasks" + dependencies = ["auth"] + + def setup(self, app): + app.include_router(router_instance.router) + + async def on_startup(self): + await cache.warm() +``` +
+ +
+πŸ” Secure Search Engine +
+ +A dynamic query builder that translates nested JSON filters into safe SQL β€” with depth-limit protection against DoS attacks. + +```json +{ + "filters": [ + { "field": "priority", "op": "eq", "value": "high" }, + { "field": "is_completed", "op": "eq", "value": false } + ], + "sort_by": "created_at", + "sort_order": "desc", + "page": 1, + "size": 20 +} +``` +
+ +
+πŸ“¦ CLI Scaffolding +
+ +Stop copying boilerplate. The `zc` CLI generates consistent, production-ready domain modules with a single command. + +| Command | Purpose | +|---------|---------| +| `zc init ` | Create a new project with settings, `.env`, and `main.py` | +| `zc startapp ` | Scaffold a domain module | +| `zc startapp --template` | Scaffold with full CRUD boilerplate | +| `zc run` | Launch Uvicorn with environment-aware config | +| `zc gensecret` | Generate a cryptographically secure `SECRET_KEY` | +
+ +--- + +## πŸ“– Documentation + +| Resource | Description | +|----------|-------------| +| [πŸš€ 10-Minute Quickstart](https://fastapi-zcore-framework.github.io/zcore/learn/quickstart/) | Build a complete Task Manager API from scratch | +| [πŸ“š Tutorial (Step-by-Step)](https://fastapi-zcore-framework.github.io/zcore/learn/overview/) | Deep dive into each architectural layer | +| [πŸ”§ How-To Guides](https://fastapi-zcore-framework.github.io/zcore/how-to/auth-setup/) | JWT auth, transactions, field masking, storage, WebSockets | +| [πŸ›οΈ Architecture & Reference](https://fastapi-zcore-framework.github.io/zcore/features/infrastructure/di/) | Full API docs for DI, Kernel, Security, Caching, and more | +| [πŸ“„ Cheat Sheet](https://fastapi-zcore-framework.github.io/zcore/cheatsheet/) | Pocket reference for quick syntax lookup | --- -## πŸ“– Documentation & Learning +## 🀝 Contributing -To explore the full capabilities of ZCore, please refer to our online documentation: +Contributions are welcome! Please read our guidelines before submitting a PR. -> ℹ️ **NOTE:** **[Getting Started Guide]** - Step-by-step tutorial to build your first service. +- **Issues:** Bug reports and feature requests via [GitHub Issues](https://github.com/fastapi-zcore-framework/zcore/issues) +- **PRs:** Open a pull request with a clear description of the change +- **Local Setup:** `pip install -e ".[all,dev]"` and run `pytest` to verify -> πŸ’‘ **TIP:** **[Architectural Concepts]** - Understand the inner mechanics of our Scoped DI, UOW, and the Core Kernel. +--- + +## πŸ“„ License -> ⚠️ **IMPORTANT:** For production environments, remember to generate a secure secret key using the `zc gensecret` CLI command and update your `.env` configuration file. +ZCore is licensed under the **Apache License 2.0**. +See [LICENSE](https://github.com/fastapi-zcore-framework/zcore/blob/master/LICENSE) for details. ---

- ZCore is licensed under the Apache License 2.0. Built with β˜• and architectural rigor. -

+ Built with β˜• and architectural rigor by Ali Alf Ostovar. +

\ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3183949 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,95 @@ +# Security Policy + +We take the security of ZCore, and the applications built on top of it, extremely seriously. Security is a core pillar of ZCore's design philosophy, integrated deeply into its architecture [security/jwt.py, security/hashing.py] rather than treated as an afterthought. + +If you believe you have found a security vulnerability in ZCore, please read this document to learn how to report it responsibly. + +--- + +## Supported Versions + +ZCore is currently in **active beta development**. Security fixes and patches are prioritized and backported to the latest release branches. + +| Version | Status | Security Support Policy | +| :--- | :--- | :--- | +| **0.1.x (Beta)** | Active | Security patches are applied to the latest active beta release | +| **< 0.1.0** | Obsolete | No security support | + +*Note: Once stable (1.y.z) releases are published, formal long-term support (LTS) security policies will be established exclusively for stable releases.* + +--- + +## Severity Classification + +To help triage reports efficiently, we classify vulnerabilities according to the following severity levels: + +| Severity | Description / Examples | +| :--- | :--- | +| **Critical** | Remote Code Execution (RCE) in ZCore core components, SQL Injection caused by a vulnerability in ZCore query generation components. | +| **High** | Authentication bypass, JWT signature forgery, or complete Scoped Context isolation failure. | +| **Medium** | Information disclosure, scoped data leakage (bypass of Zchema projection), or session fixation. | +| **Low** | Minor hardening issues, verbose error tracebacks in non-production environments. | + +--- + +## Reporting a Vulnerability + +**Please do not report security vulnerabilities via public GitHub Issues.** + +To report a vulnerability responsibly, please follow this process: + +### 1. Send a Private Report +We recommend using **GitHub Private Vulnerability Reporting** as your primary method: +* Go to the "Security" tab of this repository, select "Advisories", and click "Report a vulnerability". + +Alternatively, you can contact the maintainer directly at **`alialfostovar@gmail.com`** (the security contact listed in our package metadata). + +### 2. What to Include +To help us triage and patch the issue as quickly as possible, please include: +- A clear description of the vulnerability and its potential impact. +- Step-by-step instructions (or a minimal Proof of Concept script) to reproduce the issue. +- Details of the environment where the issue was reproduced (Python version, ZCore version, database driver). + +### 3. Disclosure Policy +We ask that you observe the principles of **Responsible Disclosure**: +- Give us reasonable time to investigate and patch the issue before making any information public. +- Do not exploit the vulnerability beyond what is strictly necessary to prove its existence. +- When applicable, security advisories may be coordinated with CVE assignment channels. + +--- + +## Our Resolution Process + +```mermaid +flowchart LR + A[Report Received] --> B[Triage & Verify
Within 48 Hours] + B --> C[Private Patch Development] + C --> D[Release Patch
Minor/Patch Release] + D --> E[Public Advisory
GitHub Security Advisory] +``` + +1. **Acknowledgment:** We will acknowledge receipt of your report within **48 hours** and begin triaging the issue. +2. **Investigation:** We will work privately to reproduce the vulnerability. If verified, we will develop a fix in a private branch. +3. **Patch Release:** We will release a patch as a new minor or patch version (e.g., `0.1.1`). +4. **Advisory:** We will publish a GitHub Security Advisory (GHSA) and coordinate CVE assignment if applicable. + +--- + +## Out of Scope + +The following are generally **not considered** security vulnerabilities in ZCore: +- **Application Misconfiguration:** Failing to configure secure environment variables (e.g., exposing database credentials). +- **Intentionally Disabled Features:** Disabling built-in security middlewares or choosing not to use Zchema projection. +- **Weak Credentials:** Brute-force or weak password choices by end-users (which are managed by application-level authentication logic). +- **Debug Mode:** Leaving FastAPI's debug or reload mode enabled in production. + +--- + +## Hardened Defaults in ZCore + +ZCore is built with strict, "secure-by-default" configurations [security/jwt.py, storage/local.py]: +- **Production Secret Guard:** The framework will raise a fatal traceback and refuse to start in a production environment if the default fallback `SECRET_KEY` is detected [security/jwt.py]. +- **Argon2id default configuration:** We enforce high-entropy password hashing out-of-the-box [security/hashing.py]. +- **Path Traversal Protection:** The local storage provider strictly verifies resolved filesystem paths before running write or unlink operations [storage/local.py]. + +*Note: ZCore only fails-fast (crashes) on fatal security configurations. Missing optional configuration elements (e.g., optional caching backends) will only raise non-fatal warnings.* \ No newline at end of file diff --git a/docs/assets/4-service.png b/docs/assets/4-service.png new file mode 100644 index 0000000..d47a565 Binary files /dev/null and b/docs/assets/4-service.png differ diff --git a/docs/assets/7-kernel.png b/docs/assets/7-kernel.png new file mode 100644 index 0000000..1a0ce39 Binary files /dev/null and b/docs/assets/7-kernel.png differ diff --git a/docs/assets/8-main.png b/docs/assets/8-main.png new file mode 100644 index 0000000..98b325b Binary files /dev/null and b/docs/assets/8-main.png differ diff --git a/docs/assets/api-router.png b/docs/assets/api-router.png new file mode 100644 index 0000000..95f83ad Binary files /dev/null and b/docs/assets/api-router.png differ diff --git a/docs/assets/auth-setup.png b/docs/assets/auth-setup.png new file mode 100644 index 0000000..26eed41 Binary files /dev/null and b/docs/assets/auth-setup.png differ diff --git a/docs/assets/cache.png b/docs/assets/cache.png new file mode 100644 index 0000000..989a69b Binary files /dev/null and b/docs/assets/cache.png differ diff --git a/docs/assets/context.png b/docs/assets/context.png new file mode 100644 index 0000000..7b62511 Binary files /dev/null and b/docs/assets/context.png differ diff --git a/docs/assets/db-events.png b/docs/assets/db-events.png new file mode 100644 index 0000000..21d6a08 Binary files /dev/null and b/docs/assets/db-events.png differ diff --git a/docs/assets/di.png b/docs/assets/di.png new file mode 100644 index 0000000..6b54118 Binary files /dev/null and b/docs/assets/di.png differ diff --git a/docs/assets/events.png b/docs/assets/events.png new file mode 100644 index 0000000..ee353f8 Binary files /dev/null and b/docs/assets/events.png differ diff --git a/docs/assets/overview.png b/docs/assets/overview.png new file mode 100644 index 0000000..37dad1b Binary files /dev/null and b/docs/assets/overview.png differ diff --git a/docs/assets/search.png b/docs/assets/search.png new file mode 100644 index 0000000..3cc4ab2 Binary files /dev/null and b/docs/assets/search.png differ diff --git a/docs/assets/security.png b/docs/assets/security.png new file mode 100644 index 0000000..1ee0ee9 Binary files /dev/null and b/docs/assets/security.png differ diff --git a/docs/assets/storage.png b/docs/assets/storage.png new file mode 100644 index 0000000..0d59c74 Binary files /dev/null and b/docs/assets/storage.png differ diff --git a/docs/assets/stream.png b/docs/assets/stream.png new file mode 100644 index 0000000..7d153c3 Binary files /dev/null and b/docs/assets/stream.png differ diff --git a/docs/assets/zchema.png b/docs/assets/zchema.png new file mode 100644 index 0000000..a3fbfc5 Binary files /dev/null and b/docs/assets/zchema.png differ diff --git a/docs/cheatsheet.md b/docs/cheatsheet.md index 5369eb5..7aec530 100644 --- a/docs/cheatsheet.md +++ b/docs/cheatsheet.md @@ -1,514 +1,404 @@ -# ZCore Framework Quick Reference (Cheat Sheet) +# Pocket Reference & Cheat Sheet -This cheat sheet serves as a quick-access reference guide for standard syntax, CLI commands, and configurations across the ZCore framework's 27 architectural layers and capabilities. +A rapid lookup guide for ZCore's core components, common patterns, and CLI commands. Each section links to the full documentation for deeper context. --- -## Block 1: πŸ–₯️ Bootstrapping & CLI +## Core Infrastructure -### 1. CLI Reference (`cli`) -The command-line utility for project scaffolding, application setup, and local development execution. +### Dependency Injection -| Command | Example Usage | Description | -| :--- | :--- | :--- | -| `init` | `zc init billing_api` | Initializes a new ZCore project workspace. | -| `startapp` | `zc startapp checkout -t` | Scaffolds a new domain application with boilerplate code. | -| `run` | `zc run` | Launches the local reload-enabled development server. | -| `gensecret` | `zc gensecret` | Generates a secure, 64-character secret key. | +| Concept | Usage | +|---|---| +| **IoCContainer** | `container = IoCContainer()` β€” Central registry for all DI lifecycles | +| **Inject** | `service: MyService = Inject(MyService)` β€” FastAPI-compatible DI wrapper | +| **Singleton** | `container.register_singleton(Interface, instance)` β€” One instance per app lifetime | +| **Scoped** | `container.register_scoped(Interface, Impl)` β€” One instance per HTTP request | +| **Transient** | `container.register_transient(Interface, Impl)` β€” New instance every resolution | +| **Reflection Cache** | `_dependency_signature_cache` β€” Caches `get_type_hints` results after first resolution | +| **Nested Dependencies** | Auto-wires recursive constructor graphs β€” `Service β†’ Repository β†’ Session` | +| **Annotated Support** | Resolves `Annotated[Type, ...]` by extracting the base type | +| **SessionDep** | `db: SessionDep` β€” `Annotated[AsyncSession, Depends(get_db)]` alias | +| **get_db** | `async def get_db()` β€” Resolves scoped `AsyncSession` from container | -πŸ‘‰ [CLI Reference Documentation](features/infrastructure/cli.md) +[Full DI Documentation](features/infrastructure/di.md) --- -### 2. Configuration (`config`) -Loads environmental configuration maps securely with fallback settings and lazy resolution through the settings proxy. +### Framework Engine (Kernel) -```python -from zcore.config import settings - -# Lazy attribute resolution avoids premature instantiation during import time -db_url = settings.DATABASE_URL -token_lifespan = settings.ACCESS_TOKEN_EXPIRE_MINUTES -``` +| Concept | Usage | +|---|---| +| **Plugin Protocol** | Class with `name`, `dependencies`, `setup()`, `on_startup()`, `on_shutdown()` | +| **Kernel** | `kernel = Kernel(); kernel.add_plugin(MyPlugin()); kernel.setup(app)` | +| **Lifespan** | `FastAPI(lifespan=kernel.lifespan)` β€” Auto-orders startup/shutdown via DAG | +| **Dependency Sort** | `graphlib.TopologicalSorter` β€” Ensures plugins init in correct order | +| **Missing Check** | Raises `RuntimeError` if a declared plugin dependency is not registered | -πŸ‘‰ [Configuration Reference Documentation](features/infrastructure/config.md) +[Full Kernel Documentation](features/infrastructure/kernel.md) --- -### 3. Application Entry (`main.py`) -Bootstrapping boilerplate that brings together logging, dbs, routers, plugins, and the core lifecycle kernel. +### Kernel Events -```python -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from fastapi import FastAPI -from zcore import Kernel, settings -from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware -from zcore.exceptions import app_exception_handler, AppException -from zcore.db import db_manager, register_db_event_dispatcher -from zcore.logging import setup_logging - -setup_logging() -db_manager.init_app(db_url=settings.DATABASE_URL) - -kernel = Kernel() -register_db_event_dispatcher(kernel.dispatcher) - -app = FastAPI(title=settings.PROJECT_NAME, lifespan=kernel.lifespan) -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) -app.add_exception_handler(AppException, app_exception_handler) - -kernel.setup(app) -``` +| Concept | Usage | +|---|---| +| **EventDispatcher** | `await dispatcher.dispatch("event.name", key=value)` | +| **Subscribe** | `@dispatcher.subscribe("event.name")` β€” Decorate any sync/async handler | +| **Concurrent Execution** | `asyncio.gather(return_exceptions=True)` β€” All async handlers run in parallel | +| **Failure Isolation** | One handler exception never blocks other handlers | -πŸ‘‰ [Application Entry Reference Documentation](features/infrastructure/main.md) +[Full Events Documentation](features/infrastructure/events.md) --- -## Block 2: 🧱 Modular App Architecture +### ASGI Middlewares -### 4. DB Infrastructure (`db`) -Coordinates the database connection engine and registers table metadata for permission checking. +| Concept | Usage | +|---|---| +| **RequestLogMiddleware** | `app.add_middleware(RequestLogMiddleware)` β€” Logs method, path, duration, correlation ID | +| **ScopedDependencyMiddleware** | `app.add_middleware(ScopedDependencyMiddleware)` β€” Initializes request-scoped DI context | +| **Correlation ID** | Auto-generated `x-request-id` header, validated against regex, returned in response | -```python -from zcore.db import db_manager - -# Initialize pool tuning and SQL query logging options -db_manager.init_app(db_url="sqlite+aiosqlite:///dev.db", pool_size=5, max_overflow=10, echo=True) -``` - -πŸ‘‰ [Database Infrastructure Reference Documentation](features/database/db.md) +[Full Middleware Documentation](features/infrastructure/middleware.md) --- -### 5. Database Model (`db models`) -SQLAlchemy 2.0 declarative database model mapping, using UUIDv4 as the default identifier. +### Application Entry (main.py) -```python -import uuid -from sqlalchemy import String, Numeric -from sqlalchemy.orm import Mapped, mapped_column -from zcore.db.setup import Base +| Concept | Usage | +|---|---| +| **Bootstrap Order** | `setup_logging()` β†’ `db_manager.init_app()` β†’ `Kernel()` β†’ `FastAPI(lifespan=...)` β†’ `kernel.setup(app)` β†’ middlewares | +| **Exception Handler** | `app.add_exception_handler(AppException, app_exception_handler)` | +| **DB-Event Bridge** | `register_db_event_dispatcher(kernel.dispatcher)` | -class Product(Base): - __tablename__ = "products" - - id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) - name: Mapped[str] = mapped_column(String(100), nullable=False) - price: Mapped[float] = mapped_column(Numeric(10, 2), nullable=False) -``` - -πŸ‘‰ [Database Model Reference Documentation](learn/1-db.md) +[Full Main Documentation](features/infrastructure/main.md) --- -### 6. Pydantic Schemas (`schemas`) -Pydantic V2 payload definitions that enforce data type casting and load attributes directly from database mappers. All schemas should inherit from `Zchema` and declare a `__db_name__` for domain-bound security. +### Configuration Settings -```python -from zcore import Zchema -from pydantic import Field, ConfigDict - -class ProductResponse(Zchema): - __db_name__ = "product" - id: str - name: str - - # Enables automatic mapping from database models - model_config = ConfigDict(from_attributes=True) -``` +| Concept | Usage | +|---|---| +| **Settings Proxy** | `from zcore import settings` β€” Lazy-resolved, no circular imports | +| **Pydantic V2** | `ZCoreCoreSettings` inherits `pydantic_settings.BaseSettings` | +| **Lazy Resolution** | `settings.DATABASE_URL` β€” Resolved from IoC container on first access | +| **Env Switching** | `ENV_FILE=.env.test` β€” Dynamic `.env` file selection | -πŸ‘‰ [Schemas Reference Documentation](learn/2-schemas.md) +[Full Config Documentation](features/infrastructure/config.md) --- -### 7. Repository Layer (`repository`) -Data access layer that provides standard CRUD query operations directly with zero SQL boilerplate. +### CLI Scaffolding -```python -from zcore.db.repository import BaseRepository -from zcore.db.setup import SessionDep -from .models import Product - -class ProductRepository(BaseRepository[Product, ProductCreate, ProductUpdate]): - def __init__(self, db: SessionDep): - super().__init__(model=Product, db=db) -``` +| Command | Description | +|---|---| +| `zc init ` | Create new project with secure `.env` and directory structure | +| `zc startapp ` | Scaffold a domain module (model, schema, repo, service, router, plugin) | +| `zc run` | Launch `uvicorn` with auto `PYTHONPATH` injection | +| `zc gensecret` | Generate a cryptographically secure 64-char secret key | -πŸ‘‰ [Repository Layer Reference Documentation](features/database/repository.md) +[Full CLI Documentation](features/infrastructure/cli.md) --- -### 8. Business Service (`service`) -The business logic layer, featuring integrated dependency injection, `on_*` persistence methods, and pre/post lifecycle hooks. +### System Utilities -```python -from zcore.service.base import BaseService -from zcore.kernel.di import Inject -from .repositories import ProductRepository -from .models import Product +| Function | Description | +|---|---| +| `json_dumps(data)` | Serialize UUID, Decimal, datetime safely | +| `json_loads(data)` | Deserialize with enhanced type support | +| `slugify(text)` | URL-safe slug: `"ZCore Framework!"` β†’ `"zcore-framework"` | +| `validate_json_schema(data, schema)` | Validate against Draft-7, raises `ValidationError` on failure | -class ProductService(BaseService[Product, ProductCreate, ProductUpdate]): - def __init__(self, repository: ProductRepository = Inject(ProductRepository)): - super().__init__(model=Product, repository=repository) -``` - -πŸ‘‰ [Business Service Reference Documentation](features/database/service.md) +[Full Utils Documentation](features/infrastructure/utils.md) --- -### 9. Routing Scaffolding (`BaseRouter`) -A router class that automatically generates and secures all standard REST endpoints. +## Data & Transactions -```python -from zcore.web import BaseRouter -from .schemas import ProductCreate, ProductUpdate, ProductResponse -from .services import ProductService -from .models import Product +### Database Manager -class ProductRouter(BaseRouter[ProductCreate, ProductUpdate]): - model = Product - create_schema = ProductCreate - update_schema = ProductUpdate - schema_out = ProductResponse - service = ProductService - prefix = "/products" -``` +| Concept | Usage | +|---|---| +| **init_app** | `db_manager.init_app(db_url="postgresql+asyncpg://...")` | +| **SQLite Bypass** | Auto-ignores `pool_size`/`max_overflow` for SQLite connections | +| **Session Context** | `async with db_manager.session() as session:` β€” Auto rollback on exception | +| **Engine Access** | `db_manager._engine` β€” Raw `AsyncEngine` for migrations | -πŸ‘‰ [Routing Scaffolding Reference Documentation](learn/5-router.md) +[Full DB Documentation](features/database/db.md) --- -### 10. Plugin System (`Plugin`) -Packages modular components to handle initialization logic and coordinate application lifecycles. +### Repository Pattern -```python -from fastapi import FastAPI -from zcore.kernel import Plugin - -class ProductPlugin(Plugin): - name = "products" - version = "1.0.0" - dependencies = [] - - def setup(self, app: FastAPI) -> None: - app.include_router(product_router) - - async def on_startup(self) -> None: - pass # Handle startup tasks (e.g. warming caches) -``` +| Concept | Usage | +|---|---| +| **BaseRepository** | `class ProductRepo(BaseRepository[Product, CreateSchema, UpdateSchema])` | +| **Read/Write Mixins** | `get()`, `create()`, `update()`, `delete()`, `get_by_ids()` | +| **Exist Optimization** | `exists(**filters)` β€” Uses `SELECT 1` for fast boolean check | +| **Search** | `search(SearchRequest)` β€” Delegates to `SearchEngine` | -πŸ‘‰ [Plugin System Reference Documentation](features/infrastructure/kernel.md) +[Full Repository Documentation](features/database/repository.md) --- -### 11. Framework Core (`engine - kernel`) -The central engine that registers, topologically sorts, and triggers application modules during server startup. - -```python -from zcore import Kernel -from products.plugin import ProductPlugin +### Business Services -kernel = Kernel() -# Register the plugin to include its modules during the startup phase -kernel.add_plugin(ProductPlugin()) -``` +| Concept | Usage | +|---|---| +| **BaseService** | `class ProductService(BaseService[Product, CreateSchema, UpdateSchema])` | +| **Safe Commit** | `await self._safe_commit()` β€” Context-aware, skips if UoW is active | +| **Lifecycle Hooks** | `after_create(obj)`, `after_update(obj)`, `after_delete(obj)` β€” Override for side effects | -πŸ‘‰ [Framework Core Reference Documentation](features/infrastructure/kernel.md) +[Full Service Documentation](features/database/service.md) --- -## Block 3: πŸ”Œ DI, Middlewares & Utilities - -### 12. Dependency Injection (`di`) -Registers and resolves classes. It supports Scoped, Transient, and Singleton lifecycles. +### Unit of Work -```python -from zcore.kernel.di import container, Inject - -# Register singleton -container.register_singleton(IMailer, SendGridMailer()) +| Concept | Usage | +|---|---| +| **UoW** | `async with UnitOfWork(db_manager) as uow:` β€” Atomic transaction boundary | +| **Post-Commit Events** | `uow.add_event("event.name", **kwargs)` β€” Dispatched only after successful commit | +| **Isolation** | Multiple repos share the same transaction within a UoW block | -# Register scoped class (request lifetime) -container.register_scoped(IOrderRepository, SQLOrderRepository) - -# Inject dependencies directly into constructors -def __init__(self, repo: IOrderRepository = Inject(IOrderRepository)): - self.repo = repo -``` - -πŸ‘‰ [Dependency Injection Reference Documentation](features/infrastructure/di.md) +[Full UoW Documentation](features/database/uow.md) --- -### 13. Kernel Events (`kernel - event`) -Subscribes to and dispatches events asynchronously, running listeners concurrently on the active event loop. +### Database Events -```python -from zcore.kernel.events import EventDispatcher +| Concept | Usage | +|---|---| +| **dispatch_db_event** | `dispatch_db_event("event.name", session, **kwargs)` β€” Error-isolated propagation | +| **Registration** | `register_db_event_dispatcher(dispatcher)` β€” Bridges DB events to app-level dispatcher | -# Subscribe a listener to an event key -dispatcher.subscribe("order.created", send_invoice_email) - -# Dispatch an event concurrently across all registered listeners -await dispatcher.dispatch("order.created", order_id="ord_102") -``` - -πŸ‘‰ [Kernel Events Reference Documentation](features/infrastructure/events.md) +[Full DB Events Documentation](features/database/db-events.md) --- -### 14. ASGI Middlewares (`middleware`) -Configures tracing and sets up request-scoped memory environments to isolate concurrent requests. +### Pagination Engine -```python -# Registered in main.py to handle request tracking and dependency isolation -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) -``` +| Concept | Usage | +|---|---| +| **Cursor Pagination** | `CursorParams(cursor="base64string", size=20)` β€” Keyset-based, stable under writes | +| **Offset Pagination** | `PageNumberParams(page=1, size=20)` β€” Standard offset/limit | +| **Base64 Encoding** | Cursor is Base64-encoded for safe URL transport | +| **Count Optimization** | `order_by(None).count()` β€” Strips ordering for faster count queries | -πŸ‘‰ [ASGI Middlewares Reference Documentation](features/infrastructure/middleware.md) +[Full Pagination Documentation](features/database/pagination.md) --- -### 15. Utilities (`utils`) -Serializes non-primitive types and validates structures using JSON standard validation rules. +### Dynamic Search Engine -```python -from zcore.utils.helpers import json_dumps -from zcore.utils.jsonschema import validate_json_schema +| Concept | Usage | +|---|---| +| **SearchRequest** | `SearchRequest(filters={...}, sorts=[...], eager_loads=[...])` | +| **JSON Query Builder** | Translates nested JSON filters to safe SQL WHERE clauses | +| **Depth Protection** | `max_depth` and `max_filters` β€” Prevents DoS via deeply nested queries | +| **Wildcard Escaping** | Auto-escapes `%` and `_` in LIKE patterns | -# Serialize UUIDs, Decimals, and Dates cleanly -serialized_payload = json_dumps({"id": uuid.uuid4(), "amount": Decimal("10.50")}) - -# Validate data structures against a Draft-7 schema -validate_json_schema(data={"age": 20}, schema={"properties": {"age": {"type": "integer"}}}) -``` - -πŸ‘‰ [Utilities Reference Documentation](features/infrastructure/utils.md) +[Full Search Documentation](features/database/search.md) --- -## Block 4: πŸ“Š Transactions, Search & Pagination +## Security & Shielding -### 16. Multi-Repo Transactions (`uow`) -Executes transactional writes across multiple repositories in a single block, ensuring atomic changes. +### Thread-Safe Context -```python -from zcore.db.uow import UnitOfWork +| Concept | Usage | +|---|---| +| **request_context** | `with request_context(user_id=..., fields={"products.cost"}):` | +| **get_current_user_id()** | Returns `uuid.UUID` from active context | +| **get_restricted_fields()** | Returns `frozenset` of restricted field paths | +| **Token Reset** | Context uses `contextvars.Token` for safe restoration | -# Wrap multiple changes in a single database transaction -async with UnitOfWork(session, dispatcher) as uow: - await order_repo.create(order_schema) - await product_repo.update(product_id, update_schema) - uow.register_event("checkout.completed", {"order_id": 102}) -``` - -πŸ‘‰ [Multi-Repo Transactions Reference Documentation](features/database/uow.md) +[Full Context Documentation](features/security/context.md) --- -### 17. Database Events (`db events`) -Signals database updates and dispatches them asynchronously, keeping transaction execution non-blocking. +### Cryptographic Protocols -```python -from zcore.db.events import dispatch_db_event +| Concept | Usage | +|---|---| +| **Password Hashing** | `get_password_hash("pw")` β€” Argon2id (64MB memory, 3 iterations, 4 threads) | +| **Token Creation** | `create_token(data={"sub": id, "scopes": [...]})` β€” Auto-detects HS256 vs RS256 | +| **Production Guard** | Fatal `RuntimeError` if `SECRET_KEY` is default in production | +| **Asymmetric Detection** | Auto-switches to RS256 if `JWT_PRIVATE_KEY`/`JWT_PUBLIC_KEY` are set | -# Safely publish database events across the system -await dispatch_db_event("db.product.created", {"id": "prod_102"}) -``` - -πŸ‘‰ [Database Events Reference Documentation](features/database/db-events.md) +[Full Security Documentation](features/security/security.md) --- -### 18. DB Pagination (`pagination`) -Loads database records using either Offset-based pagination or keyset Cursor pagination. - -```python -from zcore.db.pagination import PageNumberParams, CursorParams - -# Offset parameters -offset_params = PageNumberParams(page=1, size=20, sort_by="price", sort_order="desc") +### API Routing Mechanics -# Keyset parameters (using base64 encoded cursor positions) -cursor_params = CursorParams(cursor="eyJ2YWx1ZSI6MjAsImlkIjoyfQ==", size=25) -``` +| Concept | Usage | +|---|---| +| **ZCoreAPIRoute** | `APIRouter(route_class=ZCoreAPIRoute)` β€” Custom route handler | +| **Schema Exposure** | `?schema=true` β€” Returns context-pruned JSON-Schema | +| **Vary Header** | Auto-appends `Vary: Authorization, Cookie` on pruned responses | +| **Response Preservation** | Custom `response_class` is never silently overwritten | -πŸ‘‰ [DB Pagination Reference Documentation](features/database/pagination.md) +[Full API Router Documentation](features/security/api-router.md) --- -### 19. Dynamic Search (`search`) -Builds complex database queries dynamically, validating paths and converting types securely. - -```json -{ - "filters": [ - { "field": "stock", "op": "gt", "value": 10 }, - { "op": "or", "items": [ - { "field": "name", "op": "ilike", "value": "widget" }, - { "field": "price", "op": "le", "value": 100.00 } - ]} - ], - "sort": [{ "field": "price", "order": "asc" }], - "size": 15, - "page": 1 -} -``` +### Zchema Security & Schema Pruning -πŸ‘‰ [Dynamic Search Reference Documentation](features/database/search.md) +| Concept | Usage | +|---|---| +| **Zchema** | `class ProductBase(Zchema): __model__ = "products"` β€” Single schema, context-aware | +| **Input Pruning** | `model_validator(mode="before")` β€” Prevents mass assignment | +| **Output Pruning** | `model_serializer(mode="wrap")` β€” Strips restricted fields from responses | +| **Schema Pruning** | `__get_pydantic_json_schema__` β€” Hides restricted fields from `?schema=true` | ---- +[Full Zchema Documentation](features/security/zchema.md) -## Block 5: πŸ›‘οΈ Security, Context & Response Projection +--- -### 20. Context Management (`context`) -Stores user and field restriction parameters securely across async execution blocks. Restricted fields use the `{db_name}.{field}` syntax for domain-bound security. +### Centralized Exception Handler -```python -from zcore.context import request_context, get_current_user_id +| Exception | HTTP Code | Usage | +|---|---|---| +| `EntityNotFound` | 404 | `raise EntityNotFound("User not found")` | +| `DuplicateEntity` | 409 | `raise DuplicateEntity("Email already exists")` | +| `AuthError` | 401 | `raise AuthError("Invalid credentials")` | +| `ForbiddenError` | 403 | `raise ForbiddenError("Insufficient permissions")` | +| `ValidationError` | 400 | `raise ValidationError("Invalid input")` | -# Isolate user context and field restrictions for a specific block -with request_context(user_id=active_user_id, fields=["user.hashed_password"]): - # Retrieve the active user ID within any nested function block - current_user = get_current_user_id() -``` +All exceptions return `ResponseWrapper` envelope with `error_type` and optional `payload`. -πŸ‘‰ [Context Management Reference Documentation](features/security/context.md) +[Full Exceptions Documentation](features/security/exceptions.md) --- -### 21. Cryptographic Security (`security`) -Hashes passwords using the Argon2id algorithm and issues cryptographically signed JWT credentials. +## Advanced Ecosystem -```python -from zcore.security import get_password_hash, create_token - -# Generate a secure Argon2id password hash -hashed_password = get_password_hash("raw-password-string") +### Distributed Caching -# Issue a new signed JWT access token containing claims -token = create_token(data={"sub": "user_id_102", "role": "admin"}) -``` +| Concept | Usage | +|---|---| +| **BaseCache** | `cache = BaseCache[ProductOut](prefix="products")` | +| **Get with Validation** | `await cache.get("key", target_type=ProductOut)` β€” Auto Pydantic validation | +| **Set with TTL** | `await cache.set("key", value, ttl=300)` | +| **Fallback** | Redis β†’ local `TTLLRUCache` (maxsize=1000) on failure | +| **Eviction Loop** | Background task runs every 60s, purges expired local entries | -πŸ‘‰ [Cryptographic Security Reference Documentation](features/security/security.md) +[Full Cache Documentation](features/ecosystem/cache.md) --- -### 22. Custom Route Mechanics (`api-router`) -Enables custom routing rules, including dynamic schema exposure and native Zchema-based security. +### File Storage & Security -```python -# Enables /route/?schema=true queries on the endpoint -expose_schemas = True -``` +| Concept | Usage | +|---|---| +| **LocalStorageProvider** | `LocalStorageProvider(base_path="./uploads", validators=[...])` | +| **Validators** | `FileExtensionValidator`, `MaxFileSizeValidator`, `SafeMimeTypeValidator` | +| **Magic Bytes** | Reads first 2048 bytes, blocks PHP/HTML/MZ/Shebang patterns | +| **UUID Truncation** | Filenames use first 15 chars of `uuid4()` β€” collision-resistant | +| **Path Traversal Guard** | `is_relative_to()` check prevents `../../etc/passwd` attacks | -πŸ‘‰ [Custom Route Mechanics Reference Documentation](features/security/api-router.md) +[Full Storage Documentation](features/ecosystem/storage.md) --- -### 23. Output Projection & Security (`Zchema & response`) -ZCore's `Zchema` base class automatically prunes restricted fields across schema generation, input validation, and response serialization β€” all without web-layer middleware. - -```python -from zcore import Zchema -from zcore.web.response import ResponseWrapper - -class EmployeeResponse(Zchema): - __db_name__ = "employee" - id: str - name: str - salary: float # Automatically pruned for non-admin users +### WebSocket Streaming -# Package responses in the standardized JSON envelope -response = ResponseWrapper.success_response(data=employee_data, message="Success") -``` +| Concept | Usage | +|---|---| +| **StreamManager** | `manager = StreamManager()` β€” Per-user queue management | +| **Subscribe** | `async with manager.subscription(user_id) as queue:` β€” Context manager with auto cleanup | +| **Publish** | `await manager.publish(user_id, {"type": "update", ...})` β€” Redis PubSub + local fallback | +| **Queue Protection** | `asyncio.Queue(maxsize=100)` β€” Overflowing queues are silently dropped | +| **Horizontal Scale** | All nodes share Redis PubSub pattern `stream:user:*` | -πŸ‘‰ [Zchema Security Reference Documentation](features/security/zchema.md) +[Full Stream Documentation](features/ecosystem/stream.md) --- -### 24. Global Exceptions (`exceptions`) -Standardizes custom application exception mappings and converts errors into clean JSON error responses. - -```python -from zcore.exceptions.base import ValidationError, EntityNotFound - -# Raise to return an HTTP 400 Bad Request response containing metadata -raise ValidationError(message="Validation failed", payload={"field": "email"}) +## Quick CLI Reference -# Raise to return an HTTP 404 Not Found response -raise EntityNotFound(message="Resource not found") +```bash +zc init my_project # Create new project +zc startapp inventory # Scaffold domain module +zc run # Start dev server +zc gensecret # Generate secure secret key +zc --help # List all commands ``` -πŸ‘‰ [Global Exceptions Reference Documentation](features/security/exceptions.md) - --- -## Block 6: 🌐 Advanced Ecosystem Services +## Common Patterns -### 25. Hybrid Caching (`cache`) -Caches data using local in-memory storage, falling back cleanly to Redis connections if available. +### Scaffold a Full CRUD Module ```python -from zcore.cache import BaseCache +# 1. Model +class Product(Base): + __tablename__ = "products" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + name: Mapped[str] + +# 2. Schema +class ProductBase(Zchema): + __model__ = "products" + name: str -cache = BaseCache[UserSchema](prefix="users") +# 3. Repository +class ProductRepository(BaseRepository[Product, ProductBase, ProductBase]): + def __init__(self, db: AsyncSession): super().__init__(model=Product, db=db) -# Set cache key -await cache.set(key="user_102", value=user_data, ttl=3600) +# 4. Service +class ProductService(BaseService[Product, ProductBase, ProductBase]): + def __init__(self, repository: ProductRepository = Inject(ProductRepository)): + super().__init__(model=Product, repository=repository) -# Retrieve cached payload using Pydantic schema validation -cached_user = await cache.get("user_102", target_type=UserSchema) +# 5. Router (auto-generates 7 endpoints) +class ProductRouter(BaseRouter[ProductBase, ProductBase]): + model = Product + create_schema = ProductBase + update_schema = ProductBase + schema_out = ProductBase + service = ProductService + prefix = "/products" ``` -πŸ‘‰ [Hybrid Caching Reference Documentation](features/ecosystem/cache.md) +### Register Exception Handler ---- +```python +from zcore.exceptions import AppException, app_exception_handler +app.add_exception_handler(AppException, app_exception_handler) +``` -### 26. File Storage (`storage`) -Manages file uploads securely, protecting against path traversal and validating files based on their Magic Bytes. +### Use SessionDep ```python -from zcore.storage.local import LocalStorageProvider -from zcore.storage.validators import SafeMimeTypeValidator - -# Initialize local storage provider with image magic-byte security validators -storage = LocalStorageProvider( - base_path="./uploads", - validators=[SafeMimeTypeValidator(allowed_mimes=["image/png", "image/jpeg"])] -) +from zcore.db import SessionDep -# Upload and save file safely -saved_path = await storage.upload(file, folder="profile_pictures") +@router.get("/items") +async def list_items(db: SessionDep): + result = await db.execute(select(Item)) + return result.scalars().all() ``` -πŸ‘‰ [File Storage Reference Documentation](features/ecosystem/storage.md) - ---- - -### 27. Event Streaming (`stream`) -Streams real-time updates over WebSocket channels, using Redis PubSub to route events across instances. +### Dispatch an Event ```python -from zcore.web.streams import StreamManager +await dispatcher.dispatch("user.registered", user=user, timestamp=now) +``` -stream_manager = StreamManager() +### Use Unit of Work -# Stream real-time events to a connected client -async with stream_manager.subscription(user_id) as queue: - while True: - event = await queue.get() - await websocket.send_json(event) +```python +async with UnitOfWork(db_manager) as uow: + user = await user_repo.create(data) + await uow.add_event("user.registered", user_id=user.id) +# Event dispatched only after successful commit ``` -πŸ‘‰ [Event Streaming Reference Documentation](features/ecosystem/stream.md) - --- -!!! tip "Fast Scaffold Lookup" - Use the `zc startapp -t` CLI command to quickly generate templates pre-configured with standard models, schemas, repositories, services, and routers. \ No newline at end of file +!!! tip "Full Documentation" + Each section above links to its complete documentation page. For the tutorial, see [Learn (Tutorial)](learn/overview/). For practical recipes, see [How-To Guides](how-to/auth-setup/). \ No newline at end of file diff --git a/docs/features/database/db-events.md b/docs/features/database/db-events.md index bf7e7fd..71e8fc9 100644 --- a/docs/features/database/db-events.md +++ b/docs/features/database/db-events.md @@ -1,63 +1,87 @@ -# πŸ”” Database Lifecycle Events +# Database Event Bridge -Database lifecycle events provide a modest way to listen for database modifications and trigger external actions without cluttering your core business logic. This module acts as a bridge, connecting database operations to ZCore's central event dispatcher. +Decouple data persistence from application side-effects with secure, isolated event propagation. --- -## πŸŒ‰ The Coordination Bridge +
+
+ Type
+ Middleware Bridge +
+
+ Status
+ Core Utility +
+
+ Underlying Tech
+ asyncio / structlog +
+
+ +## The Challenge +Executing side-effects (like invalidating a cache or notifying a external service) directly after a database operation often leads to **Operational Fragility**. In standard implementations: +1. **Blocked Transactions:** If an event listener is slow or synchronous, it blocks the database session from closing, increasing connection wait times. +2. **Failure Leakage:** An exception in a "secondary" task (like a failing logging service) can propagate upwards and crash the primary request, even though the database write was successful. +3. **Tight Coupling:** The database layer becomes aware of high-level application concerns, making the architecture difficult to test or split into microservices later. + +## The ZCore Elegance +ZCore provides a **Global Event Bridge** specifically for database lifecycles. By using `dispatch_db_event`, you ensure that database events are transmitted to the application dispatcher within a secure `try/except` wrapper. This guarantees that no matter how many listeners fail or crash, the core database logic remains untouched and the primary execution flow continues. + +=== "ZCore Secure Dispatch" + :::python + from zcore.db import dispatch_db_event + + async def after_record_created(record): + # Dispatched via secure bridge + # Handler failures are logged but isolated + await dispatch_db_event( + "db.record_created", + payload={"id": record.id} + ) + +=== "Manual Unprotected Dispatch" + :::python + # Risks crashing the DB operation if a listener fails + async def after_record_created(record): + try: + await dispatcher.dispatch("db.record_created", record.id) + except Exception as e: + # You must manually remember to wrap every + # dispatch call to prevent leakage + logger.error(f"Listener failed: {e}") -To allow the database layer to communicate with other parts of your system, you must register the kernel's event dispatcher. This setup is handled automatically during the framework bootstrapping in `main.py`. - -```python -# Internal bridge mechanism -_global_dispatcher: EventDispatcher | None = None +--- -def register_db_event_dispatcher(dispatcher: EventDispatcher) -> None: - """Register the global dispatcher to bridge database events to the system.""" - global _global_dispatcher - _global_dispatcher = dispatcher -``` +## Event Propagation Flow +The DB Event Bridge acts as a non-blocking safety buffer between the low-level data layer and the high-level event subscribers. -This simple registration allows any repository or service operation to publish events that other modules (like logging or search) can listen to. +

+ Event Propagation Flow +

--- -## πŸ›‘οΈ Resilient & Async Dispatching - -Database operations must remain fast and responsive. To ensure that event handlers do not slow down your database, ZCore dispatches these events **asynchronously**. - -Furthermore, the system is designed to be resilient: if an event handler fails, the error is caught and logged, but it **will not** crash your database transaction. +## Boundaries & Integration +The DB Event system bridges the gap between infrastructure and domain logic without forcing a lock-in. -| Feature | Behavior | Benefit | -| :--- | :--- | :--- | -| **Non-blocking** | Events run in the background. | Database performance is preserved. | -| **Isolated Failures** | Handler errors are caught. | A failing log listener won't stop a product save. | -| **Type Safety** | Payloads can be any object. | Flexible data sharing between modules. | +* **Kernel Integration:** During the `main.py` bootstrap, calling `register_db_event_dispatcher(kernel.dispatcher)` links the bridge to the primary application bus [db/events.py]. +* **Decoupled Payloads:** The `payload` argument accepts any Python object or dictionary, allowing you to pass raw database IDs or fully hydrated Pydantic models to your listeners. +* **Bypass:** If you require a "Strict Dispatch" (where the primary operation *should* fail if a listener fails), you can bypass the bridge and call `kernel.dispatcher.dispatch()` directly. --- -## πŸ“ Real-world Coordination +## Under-the-Hood Spec -The event bridge is ideal for keeping disparate parts of your application in sync without them being "hard-coded" together. - -```mermaid -flowchart LR - DBOp[Database Transaction] -->|Success| Dispatch[Bridge Dispatcher] - Dispatch -->|Publish 'product.created'| Kernel[System Dispatcher] - - subgraph Listeners [Independent Subscribers] - Kernel -->|Subscriber 1| Cache[Clear Memory Cache] - Kernel -->|Subscriber 2| Search[Update Search Index] - Kernel -->|Subscriber 3| Audit[Write Audit Logs] - end -``` - ---- +### 1. The Global Dispatcher Registry +ZCore maintains a private `_global_dispatcher` reference [db/events.py]. This reference is populated once during application startup via `register_db_event_dispatcher`. This "Static Registry" pattern ensures that the database layer can access the event bus without requiring complex Dependency Injection (DI) passing through every repository. -## πŸ’‘ Engineering Insights +### 2. Async Failure Isolation +The `dispatch_db_event` function is an asynchronous wrapper [db/events.py]. It wraps the underlying dispatcher call in a comprehensive `try/except Exception` block. This is critical because it prevents "Greenlet Leakage" where an unhandled exception in an async event handler could otherwise terminate the entire coroutine stack. -!!! tip "πŸ’‘ Decoupling Your Modules" - By using lifecycle events, modules like **Search Indexing** or **Audit Logging** can "observe" database changes from a distance. Your core `ProductService` remains clean and simple because it doesn't need to know that a search index or an audit log even exists. +### 3. Structured Diagnostics +When a listener fails, the bridge uses `structlog` to output a detailed error record [db/events.py]. It captures the `event_name` and the specific `payload` that caused the failure, while setting `exc_info=True`. This allows architects to debug failing side-effects from logs without the users ever seeing an error response. -!!! info "πŸ›‘οΈ Transaction Integrity" - ZCore typically dispatches these events only after the database session has successfully committed. This ensures that you don't send a "Welcome Email" or "Order Confirmation" for a transaction that ultimately failed and was rolled back. +!!! info "Event Handlers" + Database events are typically used for "Finalized" actions. If you need to trigger logic *within* a transaction, consider using `UnitOfWork.register_event` instead, which delays the bridge dispatch until the commit is confirmed. \ No newline at end of file diff --git a/docs/features/database/db.md b/docs/features/database/db.md index 01fd635..514703f 100644 --- a/docs/features/database/db.md +++ b/docs/features/database/db.md @@ -1,77 +1,93 @@ -# πŸ—„οΈ Database Infrastructure & Connection Pool +# Database Engine & Lifecycle -ZCore provides a modest and practical database infrastructure built on top of **SQLAlchemy 2.0 (Asyncio)** and `aiosqlite`. It is designed to handle the repetitive tasks of connection pooling, transactional integrity, and permission mapping so you can focus on your data structures. +Centralized management for asynchronous connection pools and transactional session lifecycles. --- -## πŸ—οΈ The Database Manager (`DatabaseManager`) - -The central `DatabaseManager` coordinates the lifecycle of the asynchronous SQL engine. It is responsible for creating a stable connection to your database and ensuring that sessions are available when a request arrives. - -### βš™οΈ Connection Pool Configuration -To prevent resource exhaustion, ZCore automatically tunes the connection pool based on your database engine. While SQLite is ideal for development, server-based databases (like PostgreSQL) require more granular control. - -| Configuration Parameter | Purpose | SQLite Behavior | Server Databases (e.g. PostgreSQL) | -| :--- | :--- | :--- | :--- | -| **`pool_size`** | Number of persistent connections kept open. | Bypassed | Configurable (Default: `5`) | -| **`max_overflow`** | Temporary connections allowed beyond `pool_size`. | Bypassed | Configurable (Default: `10`) | -| **`pool_recycle`** | Time before a connection is refreshed to prevent timeouts. | Bypassed | Configurable (Default: `1800s`) | -| **`pool_pre_ping`** | Validates connections before use to avoid stale sockets. | βœ… Enabled | βœ… Enabled | +
+
+ Type
+ Infrastructure Layer +
+
+ Status
+ Core Requirement +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / AsyncIO +
+
+ +## The Challenge +Managing database connections in an asynchronous environment requires strict discipline. Without a centralized manager, developers often face: +- **Connection Leaks:** Forgetting to close sessions or dispose of engines, leading to exhausted database pools. +- **SQLite Incompatibility:** Passing standard pooling parameters (like `pool_size` or `max_overflow`) to a SQLite driver often raises errors or causes locking issues. +- **Manual Rollbacks:** Writing repetitive `try/except/rollback/finally` blocks in every route or service to ensure data integrity during failures. +- **DI Fragmentation:** Inconsistent methods of injecting the database session into different application layers. + +## The ZCore Elegance +The `DatabaseManager` coordinates the entire SQLAlchemy lifecycle. It provides a unified `init_app` method that intelligently configures connection pools based on the database driver and exposes a thread-safe, asynchronous session factory. When combined with `SessionDep`, it ensures every request receives a cleanly managed transaction that is automatically rolled back on failure. + +=== "ZCore Orchestration" + :::python + # 1. Initialize in main.py + from zcore.db import db_manager, SessionDep + + db_manager.init_app( + db_url="postgresql+asyncpg://user:pass@host/db", + pool_size=20, + echo=False + ) + + # 2. Inject anywhere (Router, Service, or Repo) + @router.get("/data") + async def get_data(db: SessionDep): + # db is an active AsyncSession + result = await db.execute(select(MyModel)) + return result.scalars().all() + +=== "Standard SQLAlchemy Setup" + :::python + # Manual setup requires significant boilerplate + engine = create_async_engine("...", pool_size=20) + async_session = async_sessionmaker(engine, expire_on_commit=False) + + # Dependency for FastAPI + async def get_db(): + async with async_session() as session: + try: + yield session + # Manual commit might be needed here or in service + except Exception: + await session.rollback() + raise + finally: + await session.close() + + # And you must manually handle SQLite pool errors if switching envs --- -## πŸ”„ Session Lifecycle Management - -Database sessions in ZCore follow a strict asynchronous lifecycle. This ensures that every database interaction is wrapped in a transaction that is either fully completed or safely rolled back. - -```mermaid -graph TD - Start[Request Arrives] --> Open[Open Async Session] - Open --> Execute[Execute Query] - Execute --> Success{Success?} - Success -->|Yes| Commit[Commit Changes] - Success -->|No| Rollback[Rollback Changes] - Commit --> Close[Close Session & Return to Pool] - Rollback --> Error[Raise Application Exception] - Error --> Close -``` +## Boundaries & Integration +ZCore's database layer provides a chassis for SQLAlchemy without obscuring its standard features. -!!! info "πŸ›‘οΈ Automated Rollbacks" - If an unhandled error occurs anywhere inside your business logic or repository, the `DatabaseManager` automatically triggers an asynchronous rollback. This prevents "partial data" from being saved and keeps your database consistent. +* **SQLAlchemy 2.0 Native:** The `AsyncSession` provided by `SessionDep` is a standard SQLAlchemy object. You can use any native feature (Queries, Mappers, Extensions) without restriction. +* **IoC Resolved:** The `get_db` dependency retrieves the session from the ZCore `container`. This allows you to override the database session easily during integration tests using `app.dependency_overrides`. +* **Engine Access:** While typically managed internally, the raw `AsyncEngine` is accessible via `db_manager._engine` for low-level operations like creating tables or running migration scripts. --- -## πŸ“‹ Declarative Base & Standardized Permissions +## Under-the-Hood Spec -ZCore's declarative `Base` model provides a foundation for your entities. Beyond standard SQLAlchemy features, it includes a modest utility to generate table-level permission actions automatically. - -### πŸ›‘οΈ Permission Mapping Engine (`Actions`) -For every model inheriting from `Base`, ZCore generates a set of unique permission strings. These are derived from the `__tablename__` property and provide a standardized way to handle authorization. - -```python -# Example: Using the Product model from our Quickstart -class Product(Base): - __tablename__ = "products" - ... - -# ZCore automatically generates these action keys: -# Product.actions().LISTVIEW -> "products:listview" -# Product.actions().VIEW -> "products:view" -# Product.actions().CREATE -> "products:create" -# Product.actions().UPDATE -> "products:update" -# Product.actions().DELETE -> "products:delete" -``` - ---- +### 1. The SQLite Pool Bypass +Inside `init_app`, ZCore performs a driver check: `if "sqlite" not in db_url`. If the connection string points to SQLite, ZCore silently ignores `pool_size`, `max_overflow`, and `pool_recycle` parameters [db/setup.py]. This prevents the common "Unexpected argument" errors thrown by the SQLite driver while allowing the same configuration code to work across Dev (SQLite) and Prod (PostgreSQL). -## πŸ’‘ Engineering Insights +### 2. Transactional Context Guard +The `db_manager.session()` context manager is built for safety [db/setup.py]. It wraps the session lifecycle in an `async with` block. If an unhandled exception occurs within that block, it explicitly calls `await session.rollback()` before propagating the error. This ensures that a failure in a service never leaves a transaction in a "hung" state. -!!! tip "πŸ’‘ Dependency Injection Integration" - You rarely need to interact with the `DatabaseManager` directly. By using `SessionDep`, ZCore automatically injects a ready-to-use, request-scoped session into your repositories. - ```python - def __init__(self, db: SessionDep): - self.db = db # An active, managed AsyncSession - ``` +### 3. Pre-configured Resilience +The engine is initialized with `pool_pre_ping=True` and `expire_on_commit=False` by default [db/setup.py]. `pool_pre_ping` prevents "Server has gone away" errors by testing the connection before use, while `expire_on_commit=False` allows you to access model attributes after a commit without triggering additional, often failing, lazy-load queries. -!!! note "πŸ›‘οΈ Metadata Engine" - The `Base` class stores all your table definitions in a central metadata registry. This makes it easy to integrate with migration tools like **Alembic** for managing database schema changes over time. +!!! info "Connection Recycling" + ZCore defaults `pool_recycle` to 1800 seconds (30 minutes). This prevents stale connections from being terminated by database-side timeouts, which is especially important for long-running cloud SQL instances. \ No newline at end of file diff --git a/docs/features/database/pagination.md b/docs/features/database/pagination.md index fa167f9..f3e9df3 100644 --- a/docs/features/database/pagination.md +++ b/docs/features/database/pagination.md @@ -1,85 +1,91 @@ -# πŸ“Š Database Pagination +# Page-Number & Keyset Pagination -ZCore provides two modest strategies for managing large datasets: **Offset-based** (`PageNumberPagination`) and **Keyset-based** (`CursorPagination`). Choosing the right strategy depends on your application's specific performance requirements and data stability needs. +High-performance dataset navigation with stable cursors, optimized count queries, and standardized metadata. --- -## βš–οΈ Pagination Strategies Comparison - -Selecting a strategy is a balance between flexibility and performance. Use the table below to guide your decision: - -| Metric | Offset-based (🐒) | Keyset-based (πŸš€) | -| :--- | :--- | :--- | -| **Performance** | Degrades as you go deeper into pages (`O(N)`). | Remains stable regardless of depth (`O(1)`). | -| **Data Drift Safety** | Risk of missing or seeing duplicate items. | Safe across real-time, dynamic datasets. | -| **Sorting** | Highly flexible (sort by any column). | Restricted (optimized for the cursor field). | -| **Total Count** | Included (enables "Page 1 of 10"). | Usually omitted (enables "Load More"). | -| **Ideal Use Case** | Admin tables, low-traffic dashboards. | Infinite scrolls, high-traffic mobile feeds. | - ---- - -## 1. Offset Pagination (`PageNumberPagination`) - -This is the most common approach. It mimics how humans think about booksβ€”flipping to a specific page number. - -### πŸ›‘οΈ Built-in Safety -To prevent malicious SQL injection or unexpected errors, ZCore validates that your requested sorting column actually exists on the database model before the query is executed: - -```python -# Internal validation check -valid_columns = {col.key for col in inspect(model).columns} -if params.sort_by not in valid_columns: - raise ValidationError(message=f"Invalid sort field: '{params.sort_by}'") -``` - -### πŸ“‹ Standardized Metadata -The pagination engine returns a consistent metadata object to help your frontend navigate the data: -```json -{ - "total": 150, - "page": 2, - "size": 20, - "total_pages": 8, - "has_next": true, - "has_prev": true -} -``` +
+
+ Type
+ Infrastructure Layer +
+
+ Status
+ Core Capability +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / Base64 +
+
+ +## The Challenge +Implementing pagination in FastAPI often results in "Leaky Implementation Details" and performance bottlenecks: +1. **Offset Drift:** In standard `LIMIT/OFFSET` pagination, if a new record is inserted while a user is on page 2, the same item may appear again on page 3. +2. **Count Inefficiency:** Running a `SELECT COUNT(*)` on a table with millions of rows every time a user navigates can slow down the entire database. +3. **Inconsistent Metadata:** Different endpoints returning different structures for `total`, `has_next`, or `page_size`, making frontend integration difficult. +4. **Complex Sorting:** Manually writing the SQL logic to sort by dynamic fields while ensuring stable ordering. + +## The ZCore Elegance +ZCore provides two standardized pagination engines: **Page-Number** (Offset-based) for small-to-medium datasets, and **Cursor** (Keyset-based) for high-performance, stable navigation in large datasets. Both engines are integrated into `BaseRepository` and use Pydantic models for strict parameter validation. + +=== "ZCore Repository Pagination" + :::python + from zcore.db import PageNumberParams, CursorParams + + # 1. Page-Number (Offset) Navigation + params = PageNumberParams(page=1, size=20, sort_by="created_at") + result = await repository.get_list(pagination=params) + # result.meta -> {total, page, total_pages, has_next, ...} + + # 2. Cursor (Keyset) High-Performance Navigation + cursor_params = CursorParams(size=50, cursor="encoded_string...") + result = await repository.get_list(pagination=cursor_params) + # result.meta -> {next_cursor, has_more, size} + +=== "FastAPI Raw Offset Implementation" + :::python + # Manual offset calculations and count queries + @app.get("/items") + async def get_items(page: int = 1, size: int = 20, db: AsyncSession = Depends(get_db)): + offset = (page - 1) * size + + # Manual count query + total = (await db.execute(select(func.count()).select_from(Item))).scalar() + + # Manual data query + query = select(Item).offset(offset).limit(size) + items = (await db.execute(query)).scalars().all() + + return { + "data": items, + "meta": {"total": total, "page": page, "size": size} + } --- -## 2. Keyset Pagination (`CursorPagination`) - -Keyset pagination uses "reference coordinates" to identify exactly where the last page ended. This avoids the heavy lifting of skipping over thousands of rows (offsets), making it much faster for large tables. +## Boundaries & Integration +The pagination engine is built to handle raw SQLAlchemy queries without requiring ZCore-specific models. -### πŸ“ Execution Flow -The following diagram illustrates how ZCore handles an incoming cursor to find the next set of data: +* **Select Object Support:** Both `paginate` methods accept a standard SQLAlchemy `Select` object [db/pagination.py]. You can apply complex joins and filters before passing the query to the paginator. +* **Pydantic Models:** The parameters `PageNumberParams` and `CursorParams` are standard Pydantic models. You can inherit from them to add custom filters or stricter validation. +* **Bypass:** If you require a non-standard pagination logic (e.g., specialized window functions), you can bypass the paginator and execute your own query using the repository's `db` session. -```mermaid -flowchart TD - Cursor[Incoming Base64 Cursor] -->|Decode & Parse| Decoded[Cursor ID & Value] - Decoded -->|Construct Query| Query[Filter: Column < Last Value OR Column == Last Value AND ID < Last ID] - Query -->|Sort and Limit| SQL[Compile SQL Query] - SQL -->|Fetch Size + 1| Fetch[Retrieve records] - Fetch -->|Has extra item?| Next[Serialize last item as next_cursor] -``` +--- -### πŸ”’ Cursor Serialization -For safety and ease of use, cursors are encoded as URL-safe base64 strings. A ZCore cursor payload identifies a record using two fields to ensure precision, even if multiple records share the same timestamp: +## Under-the-Hood Spec -```python -# Example of the internal coordinate payload -payload = { - "value": "2023-10-27T10:00:00", # The sort value (e.g., created_at) - "id": "uuid-123-456" # The primary key for uniqueness -} -``` +### 1. The `order_by(None)` Count Optimization +When calculating the total count for Page-Number pagination, ZCore clones the original query and strips the `ORDER BY` clause using `.order_by(None)` [db/pagination.py]. This is a critical database optimization because the database engine does not need to perform expensive sorting just to count the number of rows that match the filters. ---- +### 2. Base64 Cursor Keyset +`CursorPagination` generates an opaque, URL-safe Base64 string for navigation [db/pagination.py]. This cursor contains a JSON payload with the sorting value (e.g., a timestamp) and the primary key (UUID) to act as a tie-breaker. This ensures that even if two items have the exact same timestamp, the pagination remains stable and unique. -## πŸ’‘ Engineering Insights +### 3. Date & Time ISO Transformation +The cursor encoder automatically detects `datetime` objects [db/pagination.py]. It ensures they are converted to UTC and serialized in ISO 8601 format before being Base64 encoded. This guarantees that cursors generated on one server instance can be safely decoded on another, regardless of local server time settings. -!!! tip "πŸ’‘ Choosing the Right Tool" - If your table has fewer than 10,000 rows, **Offset Pagination** is perfectly fine and easier to implement for frontend teams. Move to **Cursor Pagination** when your data volume grows or when you need a smooth, infinite-scrolling experience. +### 4. Tie-Breaker Logic (desc/asc) +To prevent "skipped" items, the cursor query uses a tuple-comparison approach [db/pagination.py]. For a descending sort, it generates a query like: `WHERE (col < last_val) OR (col == last_val AND pk < last_pk)`. This ensures that every single record is visited in a deterministic order. -!!! info "πŸ›‘οΈ Drift Protection" - Data "drift" happens when a new item is added to Page 1 while a user is browsing Page 2. In Offset pagination, this causes an item to "shift" and appear again on Page 2. Cursor pagination prevents this because it anchors the query to a specific record, not a page number. +!!! info "Cursor UI" + Because cursors are opaque strings, they are ideal for "Infinite Scroll" or "Load More" UI patterns. Use Page-Number pagination only when the user needs to jump to a specific numbered page (e.g., "Go to page 15"). \ No newline at end of file diff --git a/docs/features/database/repository.md b/docs/features/database/repository.md index 379b488..b63574d 100644 --- a/docs/features/database/repository.md +++ b/docs/features/database/repository.md @@ -1,93 +1,92 @@ -# πŸ›οΈ Repository Pattern +# Repository Pattern -ZCore uses the **Repository Pattern** to isolate data access logic from your core business logic. This modest abstraction ensures that your database interactions are consistent, reusable, and easy to test, regardless of how complex your queries become. +Decouple business logic from database execution with high-performance, type-safe, and reusable data interfaces. --- -## πŸ“ Generic Repository Architecture - -The `BaseRepository` is built using specialized mixins. This design allows you to understand exactly what capabilities a repository has based on the mixins it inherits. - -```mermaid -flowchart TD - AbstractRepository[AbstractRepository] --> ReadMixin[ReadRepositoryMixin] - AbstractRepository --> WriteMixin[WriteRepositoryMixin] - AbstractRepository --> SearchMixin[SearchRepositoryMixin] - - ReadMixin --> BaseRepository[BaseRepository] - WriteMixin --> BaseRepository - SearchMixin --> BaseRepository - -``` - ---- - -## πŸ” 1. Read Operations (`ReadRepositoryMixin`) - -The read mixin manages safe and optimized database queries. It handles everything from simple lookups to complex lists. - -### ⚑ Fast-Path Optimization -Database round-trips can be expensive. ZCore includes a modest optimization in `get_by_ids`: if an empty list is passed, it bypasses the database entirely and returns an empty sequence, saving unnecessary network latency. - -```python -async def get_by_ids(self, ids: List[Any], ...) -> Sequence[ModelType]: - # Fast path: Skip database hit entirely if input is empty - if not ids: - return [] +
+
+ Type
+ Data Access Layer +
+
+ Status
+ Recommended +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / Pydantic V2 +
+
+ +## The Challenge +In many FastAPI projects, database queries are scattered directly inside route handlers. This leads to: +- **Repetitive Query Logic:** Manually writing `session.execute(select(...))` and `.scalars().all()` for every endpoint. +- **Leaky Abstractions:** Changing a column name in the database forces you to find and update logic in dozens of router files. +- **Testing Hardship:** Mocking a raw `AsyncSession` with complex chain calls is significantly harder than mocking a high-level `get_by_id` method. +- **Inconsistent Pagination:** Different developers implementing offset or cursor logic differently, leading to unpredictable API behavior. + +## The ZCore Elegance +The `BaseRepository` provides a robust, pre-optimized set of CRUD and search operations. It leverages Python generics to ensure type safety across your domain and integrates seamlessly with ZCore’s specialized search and pagination engines. By inheriting from the repository, you gain standardized methods for batch operations, partial updates, and optimized existence checks out of the box. + +=== "ZCore Repository" + :::python + from zcore import BaseRepository + from .models import User + from .schemas import UserCreate, UserUpdate + + class UserRepository(BaseRepository[User, UserCreate, UserUpdate]): + def __init__(self, db: AsyncSession): + super().__init__(model=User, db=db) + + # Custom logic remains clean and isolated + async def get_active_users(self): + query = self._get_base_query().where(User.is_active == True) + result = await self.db.execute(query) + return result.scalars().all() - query = select(self.model).where(self.pk.in_(ids)) - result = await self.db.execute(query) - return result.scalars().all() -``` +=== "Manual SQLAlchemy Implementation" + :::python + # Repetitive boilerplate required for every entity + async def get_user(db: AsyncSession, user_id: uuid.UUID): + query = select(User).where(User.id == user_id) + result = await db.execute(query) + return result.scalars().first() + + async def update_user(db: AsyncSession, user_id: uuid.UUID, data: UserUpdate): + query = select(User).where(User.id == user_id) + user = (await db.execute(query)).scalars().first() + if user: + for key, value in data.model_dump().items(): + setattr(user, key, value) + await db.flush() + return user --- -## πŸ“ 2. Write Operations (`WriteRepositoryMixin`) - -The write mixin handles the creation, modification, and removal of database records. - -### πŸ”„ Flush Over Commit (UoW Harmony) -To coordinate with ZCore's **Unit of Work** pattern, repositories do not call `commit()` directly. Instead, they use `flush()`. This pushes your changes to the database's transaction log and retrieves auto-generated fields (like IDs) without ending the transaction. This allows the Unit of Work to decide when the final "Save" happens. +## Boundaries & Integration +ZCore Repositories are designed to be thin wrappers that expose the full power of SQLAlchemy. -| Method | Database Action | Lifecycle Behavior | -| :--- | :--- | :--- | -| ✨ `create` | `INSERT` | Flushes and refreshes the instance. | -| πŸ› οΈ `update` | `UPDATE` | Applies partial or full patches to a record. | -| πŸ—‘οΈ `delete` | `DELETE` | Removes the record from the session context. | +- **Standard AsyncSession:** The `self.db` attribute is a standard `AsyncSession`. You can execute any raw SQL or complex SQLAlchemy expression whenever the scaffolded methods are insufficient. +- **Query Extensibility:** Use `_get_base_query()` to define default filters (e.g., Soft Delete logic) and `_extend_query()` for custom joins that should apply to all read operations. +- **Eager Loading:** All read methods (`get`, `get_list`, etc.) accept an `options` parameter for standard SQLAlchemy loader options like `joinedload` or `selectinload`. +- **Selective Loading:** The `fields` parameter allows you to perform `load_only` queries, ensuring you only fetch the data you actually need from the wire. --- -## πŸ§ͺ 3. Dynamic Search (`SearchRepositoryMixin`) +## Under-the-Hood Spec -This mixin bridges your repository with ZCore's **Search Engine**. It allows you to perform complex, policy-validated searches with minimal code. +### 1. Automatic PK Inspection +Upon initialization, `BaseRepository` uses the SQLAlchemy `inspect` module to automatically identify the model's primary key [db/repository.py]. It maps this to `self.pk` and `self.pk_name`, allowing the generic CRUD methods to work regardless of whether your primary key is named `id`, `uuid`, or a custom field. -```python -async def search(self, search_in: SearchRequest, pagination: Any = None) -> Any: - from zcore.db.search import SearchEngine - engine = SearchEngine(self.model) - query = engine.build_base_query(search_in) - - if pagination is None: - result = await self.db.execute(query) - return result.scalars().all() - - # Automatically chooses Cursor or Page pagination based on params - paginator = CursorPagination(self.cursor_field) if isinstance(pagination, CursorParams) else PageNumberPagination() - return await paginator.paginate(self.db, query, pagination, self.model) -``` - ---- +### 2. The `get_by_ids` Fast Path +The `get_by_ids` implementation includes a high-performance "fast path" [db/repository.py]. If an empty list is passed as the `ids` parameter, the repository returns an empty sequence immediately without making a network round-trip to the database engine. -## πŸ’‘ Engineering Insights +### 3. Optimized Existence Checks +The `exist()` method is optimized for minimal database overhead [db/repository.py]. Instead of loading a full model instance, it executes a `select(1).where(...)` with a `limit(1)` clause, which is significantly faster and consumes less memory on both the application and database server. -!!! info "πŸ›‘οΈ Type Safety & Generics" - By utilizing Python Generics (`BaseRepository[Model, CreateSchema, UpdateSchema]`), ZCore provides full IDE auto-completion. This ensures that you don't accidentally pass an `OrderUpdate` schema to a `ProductRepository`. +### 4. Flush-Oriented Mutations +All write operations (`create`, `update`, `delete`) in the repository use `await self.db.flush()` and `await self.db.refresh()` instead of `commit()` [db/repository.py]. This ensures that changes are validated by the database but remain part of an open transaction, allowing the `UnitOfWork` to manage the final atomic commit boundary. -!!! tip "πŸ’‘ Custom Query Extensions" - If you need highly specialized SQL (e.g., complex reporting or raw queries), we suggest extending the repository class in your domain module. This keeps all SQL-related code in one place, separate from your business rules. - ```python - class ProductRepository(BaseRepository[...]): - async def get_out_of_stock_by_category(self, category_id: int): - # Write your custom SQLAlchemy query here - ... - ``` +!!! info "Cursor Field" + By default, the repository uses the primary key as the `cursor_field` for keyset pagination. You can override this in your subclass (e.g., `self.cursor_field = "created_at"`) to support stable pagination on chronological data. \ No newline at end of file diff --git a/docs/features/database/search.md b/docs/features/database/search.md index f88dea3..e30081a 100644 --- a/docs/features/database/search.md +++ b/docs/features/database/search.md @@ -1,94 +1,111 @@ -# πŸ” Dynamic Search Engine +# Dynamic Search Engine -ZCore’s search engine is a modest tool designed to handle the heavy lifting of data discovery. It translates client-supplied criteriaβ€”like nested filters, sorting rules, and relationship preloadingβ€”into safe and performant database queries without requiring you to write custom SQL for every request. +Securely translate structured JSON filters, complex sorting, and relational eager-loading into optimized SQLAlchemy queries. --- -## πŸ›‘οΈ Safety & Security Layers +
+
+ Type
+ Query Utility +
+
+ Status
+ Recommended +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / Pydantic +
+
+ +## The Challenge +Building a truly dynamic search endpoint (e.g., `POST /products/search`) in standard FastAPI is a security and maintenance nightmare. Developers often face: +1. **Manual Filter Mapping:** Writing endless `if` statements for every possible combination of `price_gt`, `category_eq`, or `name_ilike`. +2. **Unsafe Joins:** Manually managing table joins for relational filters, often leading to duplicate rows or Cartesian products. +3. **Recursive DoS:** Deeply nested "OR" and "AND" JSON filters that can crash a database if the query complexity isn't limited. +4. **Security Leaks:** Clients accidentally filtering or sorting by restricted fields (like `margin` or `internal_id`) that should be hidden from their role. + +## The ZCore Elegance +The `SearchEngine` provides a secure, JSON-driven query builder. It parses a `SearchRequest` object containing nested filters, sorting rules, and relation-preload paths. It automatically coerces types, validates the request against active context restrictions, and limits the query depth to prevent resource exhaustion. + +=== "ZCore Dynamic Search" + :::python + from zcore.db import SearchRequest, SearchEngine + + # 1. Define a structured search payload + request = SearchRequest( + filters=[ + {"field": "price", "op": "gt", "value": 100}, + {"op": "or", "items": [ + {"field": "category.name", "op": "eq", "value": "Electronics"}, + {"field": "tags", "op": "ilike", "value": "premium"} + ]} + ], + include=["category", "tags"], + sort=[{"field": "price", "order": "desc"}] + ) + + # 2. Build and execute securely via Repository + results = await product_repository.search(request) + +=== "FastAPI Raw Manual Filtering" + :::python + # Replicating dynamic logic requires hundreds of lines: + @router.get("/search") + async def manual_search( + price_gt: float = None, + category: str = None, + db: AsyncSession = Depends(get_db) + ): + query = select(Product) + if price_gt: + query = query.where(Product.price > price_gt) + if category: + query = query.join(Category).where(Category.name == category) + + # Manually handling nested OR/AND is almost impossible + # to write safely and generically for every endpoint. + result = await db.execute(query) + return result.scalars().all() -Dynamic query interfaces can be risky if not properly guarded. To prevent SQL injection, performance exhaustion, or unauthorized data access, the `SearchEngine` implements several engineering safeguards. - -```mermaid -flowchart TD - Request[Incoming SearchRequest] -->|Security Check| Guard[Check Restricted Fields] - Guard -->|Forbidden Access| Deny[Raise ForbiddenError] - Guard -->|Safe| DepthCheck{Join Depth <= 3?} - DepthCheck -->|No| Reject[Raise ValidationError] - DepthCheck -->|Yes| Coerce[Coerce Value Types] - Coerce -->|Sanitize SQL| Build[Compile Optimized SQL] -``` - -### 1. Restricted Field Guard -Before a query is even compiled, the engine cross-references all requested fields and paths against the active security context. If a user is not authorized to see a field (e.g., `internal_cost` or `hashed_password`), the engine blocks the query immediately. - -```python -# Internal logic: matches paths against domain-bound restricted fields (e.g., 'user.email', 'product.price') -if normalized_path == normalized_restricted or normalized_path.startswith(normalized_restricted + "."): - return True # Block access -``` +--- -### 2. Relationship Depth Limits -Deep joins (e.g., fetching a product's category's owner's profile) can significantly slow down a database. To maintain stability, ZCore limits relationship preloading to a maximum depth of **3 levels**. +## Search Workflow +The `SearchEngine` validates and compiles the request through a strict security-first pipeline. -### 3. Wildcard Sanitization -To prevent "expensive" string searches that can lock up database resources, ZCore automatically escapes SQL wildcards (`%` and `_`) provided by the client. This ensures that `ilike` operations behave exactly as expected without unintended side effects. +

+ Search Workflow +

-### 4. Smart Type Coercion -Since search parameters arrive as JSON (strings, numbers, or booleans), the engine intelligently converts them to the correct Python types required by your database columns, such as `UUID`, `datetime`, or `Decimal`. --- -## πŸ§ͺ Search Request Structure - -The `SearchRequest` is a structured way for clients to ask for exactly what they need. It is divided into four main parts: - -| Component | Purpose | Example | -| :--- | :--- | :--- | -| **`filters`** | Logical conditions for finding data. | `price > 100 AND stock > 0` | -| **`include`** | Related data to "eager load" in one go. | `["category", "tags"]` | -| **`sort`** | The order of the results. | `[{"field": "created_at", "order": "desc"}]` | -| **`pagination`** | How many items to return and where to start. | `size: 20, page: 1` | - -### Example Payload -```json -{ - "filters": [ - { "field": "stock", "op": "gt", "value": 0 }, - { - "op": "or", - "items": [ - { "field": "name", "op": "ilike", "value": "coffee" }, - { "field": "tags.name", "op": "eq", "value": "organic" } - ] - } - ], - "include": ["category"], - "sort": [{ "field": "price", "order": "asc" }] -} -``` +## Boundaries & Integration +ZCore provides a dynamic bridge without locking you out of SQLAlchemy's native power. ---- +* **Select Object Return:** The core engine method `build_base_query()` returns a standard SQLAlchemy `Select` object. You can still apply additional `.where()` clauses or `.limit()` before execution. +* **Custom Handlers:** If a specific field requires complex custom logic (e.g., a full-text search vector), you can register a custom handler: `engine.register_handler("full_text", my_handler_func)`. +* **Bypass:** If you only need simple primary key lookups, use the repository's `get()` or `get_by_ids()` methods instead of the search engine to save processing overhead. -## πŸ› οΈ Supported Operators +--- -The search engine supports a practical set of operators to handle most real-world filtering needs: +## Under-the-Hood Spec -| Operator | SQL equivalent | Description | -| :--- | :--- | :--- | -| `eq` / `ne` | `=` / `!=` | Equals or Not-Equals. | -| `gt` / `lt` | `>` / `<` | Greater-than or Less-than. | -| `ge` / `le` | `>=` / `<=` | Greater-or-equal or Less-or-equal. | -| `ilike` | `ILIKE` | Case-insensitive partial string match. | -| `in` | `IN (...)` | Matches any value in a provided list. | -| `is_null` | `IS NULL` | Checks if a field is empty (or not). | -| `or` / `and` | `OR` / `AND` | Logical grouping for complex filters. | +### 1. Recursive Depth Protection +To defend against "Recursive DoS" attacks, the `SearchEngine` enforces a `max_depth` limit (default: 3) [db/search.py]. If a client sends a JSON payload with more than 3 levels of nested `or`/`and` operators, the engine raises a `ValidationError` before any SQL is generated. ---- +### 2. Intelligent Loader Strategy +The engine automatically chooses the most efficient SQLAlchemy loader [db/search.py]. It inspects the model relations: +- **`joinedload`**: Used for many-to-one or one-to-one relationships to minimize queries. +- **`selectinload`**: Used for one-to-many or many-to-many collections to avoid Cartesian products and "N+1" problems. -## πŸ’‘ Engineering Insights +### 3. Safe Type Coercion +Since JSON sends everything as primitives (strings/numbers), the engine uses model inspection to coerce values [db/search.py]. It automatically detects if a target column is a `uuid.UUID`, `date`, or `datetime` and attempts to parse the incoming string using ISO 8601 standards before creating the comparison expression. -!!! tip "πŸ’‘ Why use `include`?" - Using the `include` field allows you to fetch related data in a single database round-trip. ZCore uses **Joined Loading** for single items and **Select-In Loading** for collections, ensuring the most efficient query pattern is used automatically. +### 4. Wildcard Sanitization +For `ilike` operations, ZCore automatically escapes SQL-sensitive wildcards (`%`, `_`, `\`) [db/search.py]. This prevents clients from performing "blind" wildcard searches that could lead to full-table scans and performance degradation. -!!! info "πŸ›‘οΈ Access Denied?" - If you receive a `ForbiddenError` during a search, it is likely because one of the fields in your `filters`, `sort`, or `include` has been restricted in the current request context. This is ZCore's way of preventing accidental data exposure. +!!! info "Security Isolation" + The SearchEngine is strictly integrated with `zcore.context`. If a field dot-path (e.g., `category.secret_code`) is restricted in the active user context, any attempt to filter, sort, or include that field will result in a `ForbiddenError`. \ No newline at end of file diff --git a/docs/features/database/service.md b/docs/features/database/service.md index a93b2e4..5e55452 100644 --- a/docs/features/database/service.md +++ b/docs/features/database/service.md @@ -1,108 +1,103 @@ -# βš™οΈ Business Logic Layer (Base Service) +# Business Service Orchestration -The service layer is the modest heart of your application's logic. It acts as a bridge between the raw data access of the Repository and the web requests of the Router. Its primary job is to orchestrate business rules and ensure that database transactions are handled safely and atomically. +Coordinate domain logic with transactional safety and granular lifecycle hooks. --- -## 🎣 Base Service Lifecycle Hooks - -ZCore's `BaseService` provides built-in "hooks" that run automatically around database modifications. These hooks allow you to validate, clean, or enrich your data without cluttering the core logic of saving to the database. - -The public orchestrator methods (`create`, `create_multi`, `update`, `update_multi`, `delete`, `delete_multi`) follow the **Template Method** pattern. They coordinate the execution of `pre_*` hooks, the customizable `on_*` database action, `post_*` hooks, and the `_safe_commit` boundary β€” in that exact order. - -```mermaid -flowchart TD - Input[Incoming Request Payload] --> Pre[1. pre_create / pre_update / pre_delete] - Pre -->|Business Validation| On[2. on_create / on_update / on_delete] - On -->|Custom Persistence Logic| Post[3. post_create / post_update / post_delete] - Post -->|Prepares Events| Commit[4. _safe_commit] -``` - -### πŸ“‹ Available Lifecycle Hooks - -| Hook | Phase | Primary Use Case | -| :--- | :--- | :--- | -| ✨ `pre_create` | Before SQL Insert | Data sanitization and domain-level validation. | -| βœ… `on_create` | Core database insert | Override to customize the creation logic. | -| βœ… `post_create` | After SQL Insert | Triggering side effects like logging or metrics. | -| πŸ› οΈ `pre_update` | Before SQL Update | Checking permissions or calculating derived fields. | -| πŸ”„ `on_update` | Core database update | Override to customize the update logic. | -| πŸ”„ `post_update` | After SQL Update | Invalidating caches or notifying other systems. | -| πŸ—‘οΈ `pre_delete` | Before SQL Delete | Verifying that the record can be safely removed. | -| πŸ—‘οΈ `on_delete` | Core database delete | Override to customize the deletion logic. | - -!!! tip "🧠 Why `on_*` Instead of Overriding `create()`?" - Previously, developers overrode `create()` directly, which risked skipping hooks or the safe commit boundary. The new `on_*` methods are the **only** extension points you need for custom database/persistence logic. The orchestrator guarantees that `pre_*` hooks run first, your custom logic runs next, `post_*` hooks run after, and the transaction is committed safely β€” every time. +
+
+ Type
+ Domain Logic Layer +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ Python / Unit of Work +
+
+ +## The Challenge +When business logic is scattered across routers, applications become fragile and hard to test. Developers struggle with: +- **Transaction Splitting:** Sub-operations accidentally committing early, leaving the database in a partial state if a subsequent step fails. +- **Hook Spaghetti:** Manually triggering side effects (like sending an email or clearing a cache) inside every route handler. +- **Redundant Error Checks:** Repeating `if not item: raise HTTPException(404)` in every single endpoint. +- **Tight Coupling:** Business rules depending directly on HTTP request objects, making them impossible to run in background tasks or CLI scripts. + +## The ZCore Elegance +The `BaseService` acts as a pure business orchestrator. It wraps repository calls in a strict **Lifecycle Hook** pipeline and implements the **Safe Commit** protocol. This ensures that domain events are triggered correctly and transactions are only finalized if they are not part of a larger, external `UnitOfWork`. + +=== "ZCore Service Orchestration" + :::python + from zcore import BaseService, Inject + from .repositories import OrderRepository + from .models import Order + + class OrderService(BaseService[Order, OrderCreate, OrderUpdate]): + def __init__(self, repo: OrderRepository = Inject(OrderRepository)): + super().__init__(model=Order, repository=repo) + + async def post_create(self, model: Order): + # Side effect triggered automatically and safely + await notify_service.send_order_confirmation(model) + + async def pre_update(self, id, schema, partial): + # Domain validation before DB hit + if schema.status == "cancelled" and not user.is_admin: + raise ForbiddenError("Only admins can cancel orders") + +=== "FastAPI Manual Orchestration" + :::python + # Logic mixed with HTTP concerns in the router + @router.post("/") + async def create_order(data: OrderCreate, db: AsyncSession = Depends(get_db)): + # 1. Manual Validation + if data.amount <= 0: raise HTTPException(400) + + # 2. Manual DB Logic + order = Order(**data.model_dump()) + db.add(order) + try: + await db.commit() # Dangerous: might break outer transactions + except Exception: + await db.rollback() + raise + + # 3. Manual Side Effect (prone to being forgotten) + await notify_service.send_confirmation(order) + return order --- -## πŸ›‘οΈ Transaction Isolation & Safe Commits +## Boundaries & Integration +ZCore Services are framework-agnostic units that coordinate between the Web and Data layers. -One of the most challenging problems in complex systems is the **Nested Commit Problem**. If multiple services each try to "save" (commit) their own work, it becomes impossible to run them all together in a single atomic transaction. If the second service fails, the first one has already saved its data, leading to a corrupted state. - -ZCore solves this modestly through the `_safe_commit` mechanism. This helper checks if the current operation is part of a larger **Unit of Work**: - -```python -async def _safe_commit(self) -> None: - session_info = self.repository.db.info - - # If the session is wrapped in a UnitOfWork, we stay quiet and let UOW handle the commit. - if not session_info.get("uow_managed", False): - try: - await self.repository.db.commit() - except Exception: - await self.repository.db.rollback() - raise -``` - -### πŸ’‘ How it works in practice: - -* **Standalone Mode**: When you call a service method normally, it commits its own changes immediately. -* **Orchestrated Mode**: When you call multiple service methods inside a `UnitOfWork` block, ZCore sets `uow_managed` to `True`. The services skip their individual commits, allowing the Unit of Work to commit everything at once at the very end. +- **Web Independent:** Services never accept `Request` objects or return `JSONResponse`. They operate strictly on Pydantic schemas and SQLAlchemy models, allowing them to be called from Routers, Background Tasks, or CLI commands. +- **Service-to-Service DI:** Use `Inject()` in the constructor to pull in other domain services. ZCore's DI container handles the resolution and ensures correct lifecycles. +- **Hook Bypass:** If you need to perform an operation without triggering hooks (e.g., for data migrations), you can call the "raw" methods like `on_create` or `on_update` directly. +- **Transactional Unity:** Services automatically detect if they are running inside a `UnitOfWork` block and will defer commits to the outer orchestrator. --- -## πŸ’» Practical Usage - -When building your services, you can easily tap into these hooks to enforce business rules: - -```python -from zcore.service.base import BaseService -from zcore.exceptions.base import ValidationError -from products.schemas import ProductCreate, ProductUpdate -from products.models import Product - -class ProductService(BaseService[Product, ProductCreate, ProductUpdate]): - - async def pre_create(self, schema: ProductCreate) -> None: - """Enforce business rules before data reaches the repository.""" - if schema.price > 10000 and schema.stock < 5: - raise ValidationError( - message="High-value products must be initialized with a minimum of 5 items." - ) - - async def on_create(self, schema: ProductCreate) -> Product: - """Custom database/persistence logic for single-record creation. - - Override this instead of the public `create()` orchestrator. - The orchestrator guarantees that pre/post hooks and the safe commit - boundary are always executed around this method. - """ - # Apply any custom transformations before persisting - return await self.repository.create(schema) - - async def post_create(self, model: Product) -> None: - """Logic to run after the database has flushed the new record.""" - # This is a safe place to prepare a 'product.created' event. - pass -``` +## Under-the-Hood Spec ---- +### 1. The Safe Commit Protocol +Every mutation method in `WriteServiceMixin` concludes with `await self._safe_commit()` [service/base.py]. This method inspects the database session's internal `info` dictionary for the `uow_managed` flag. If a `UnitOfWork` context is active, the service **refrains from committing**, preserving the atomicity of the larger transaction. + +### 2. Execution Hook Pipeline +ZCore enforces a strict sequence for all write operations. For example, the `create` method follows this path [service/base.py]: +1. `pre_create(schema)`: Validation or data enrichment. +2. `on_create(schema)`: The actual repository write (Flush). +3. `post_create(model)`: Side effects like logging or events. +4. `_safe_commit()`: Final transactional boundary check. -## πŸ’‘ Engineering Insights +### 3. Automated Error Mapping +The `ReadServiceMixin` intercepts empty repository results and automatically raises an `EntityNotFound` exception [service/base.py]. This ensures that your API consistently returns a standardized 404 response through the global exception handler without requiring manual checks in your business code. -!!! tip "πŸ’‘ Validation vs. Sanitization" - Use `pre_create` for **Business Logic Validation** (e.g., "Is this user allowed to buy this?"). Use Pydantic schemas for **Structural Validation** (e.g., "Is this field an integer?"). This keeps your service layer focused on rules rather than data types. +### 4. Post-Read Interceptors +Read operations (`get`, `get_list`) include `post_get` and `post_get_multi` hooks [service/base.py]. These are ideal for calculating dynamic properties or performing permission-based masking on model attributes before they reach the web serialization layer. -!!! info "πŸ›‘οΈ Automatic Rollbacks" - If an exception is raised inside a `pre_` hook or during the `on_` database action, ZCore will prevent the commit and ensure the database session is rolled back. Your data remains safe and consistent. \ No newline at end of file +!!! info "Post-Search Hook" + The `SearchServiceMixin` also includes a `post_search` hook, allowing you to batch-process results from complex dynamic queries before returning them to the client. \ No newline at end of file diff --git a/docs/features/database/uow.md b/docs/features/database/uow.md index ec1be1f..a65d892 100644 --- a/docs/features/database/uow.md +++ b/docs/features/database/uow.md @@ -1,102 +1,96 @@ -# πŸ”— Unit of Work (UOW) Pattern +# Unit of Work (Atomic Transactions) -The Unit of Work (UOW) pattern is a practical way to coordinate multiple database changes and manage system-wide events. It ensures that complex operations involving several services or repositories succeed or fail as a single, atomic unit, while preventing "partial saves" that could corrupt your data. +Coordinate multiple domain operations into a single atomic transaction with safe, post-commit event dispatching. --- -## πŸ“ Transactional Lifecycle Flow - -The `UnitOfWork` class manages the database session and maintains a modest internal queue (`_pending_events`). This queue buffers domain events (like sending a "Welcome Email") so they are only triggered **after** the data is safely committed to the database. - -```mermaid -sequenceDiagram - participant App as Business Logic - participant UOW as UnitOfWork - participant DB as Database Session - participant ED as EventDispatcher - - Note over App,UOW: Enter 'async with uow:' block - UOW->>DB: Set uow_managed = True - - App->>DB: Modify Records (via Repository) - DB-->>App: Changes Flushed (Not Saved Yet) - - App->>UOW: register_event("order.placed", data) - Note over UOW: Event stored in _pending_events - - Note over App,UOW: Exit block (No Errors) - UOW->>DB: commit() - DB-->>UOW: Success - - Note over UOW: Data is safe. Dispatching events... - loop Post-Commit Dispatch - UOW->>ED: dispatch("order.placed", data) - end -``` +
+
+ Type
+ Transaction Orchestrator +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / asyncio +
+
+ +## The Challenge +In complex business flows (e.g., a **Checkout** involving Inventory, Payment, and Shipping), maintaining data integrity is difficult. Standard implementations often suffer from: +1. **Premature Commits:** A sub-service (e.g., `InventoryService`) commits its changes independently. If the subsequent `PaymentService` fails, the inventory is already deducted, leaving the system in an inconsistent state. +2. **Ghost Notifications:** An event (like a "Welcome Email") is dispatched as soon as a user is created. If the database transaction later fails and rolls back, the user receives an email for an account that doesn't exist. +3. **Boilerplate Fatigue:** Wrapping every multi-step operation in `try/except/rollback` blocks leads to noisy, error-prone code. + +## The ZCore Elegance +The `UnitOfWork` (UOW) acts as the single source of truth for a transaction's lifecycle. It marks the database session as "Managed," signaling to all internal ZCore services that they must defer their commits. Furthermore, it provides an event buffer that guarantees domain events are **only dispatched after** a successful database commit. + +=== "ZCore Unit of Work" + :::python + from zcore import UnitOfWork, Inject + + async def checkout_flow( + data: CheckoutSchema, + order_service: OrderService = Inject(OrderService), + inventory_service: InventoryService = Inject(InventoryService), + dispatcher: EventDispatcher = Inject(EventDispatcher), + session: AsyncSession = SessionDep + ): + # 1. Atomic context manager + async with UnitOfWork(session, dispatcher) as uow: + # Sub-services detect UOW and skip internal commits + order = await order_service.place_order(data) + await inventory_service.reduce_stock(order.items) + + # 2. Register event for post-commit dispatch + uow.register_event("order.placed", {"id": order.id}) + + # 3. Auto-commit on exit / Auto-rollback on exception + +=== "Raw SQLAlchemy Implementation" + :::python + async def checkout_flow(data, db, dispatcher): + try: + # 1. Manual order logic + order = await create_order(db, data) + # 2. Manual inventory logic + await reduce_stock(db, order.items) + + # 3. Manual commit + await db.commit() + + # 4. Manual event dispatch (Risk: if commit failed, + # or if this fails, logic becomes messy) + await dispatcher.dispatch("order.placed", {"id": order.id}) + except Exception: + # 5. Manual rollback + await db.rollback() + raise --- -## πŸ› οΈ Automated Transactional Boundaries +## Boundaries & Integration +ZCore’s UOW is a high-level orchestrator that respects underlying SQLAlchemy protocols. -ZCore utilizes the asynchronous context manager protocol (`async with`) to automate the start and end of your transactions. This removes the need for manual `commit()` or `rollback()` calls in your business logic. - -| Phase | Behavior | Purpose | -| :--- | :--- | :--- | -| 🏁 **Entry** | Sets `uow_managed = True`. | Tells internal services to skip their individual commits. | -| πŸ›‘ **Exception** | Triggers `rollback()`. | Discards all changes if any part of the block fails. | -| βœ… **Success** | Triggers `commit()`. | Saves all changes to the database permanently. | -| πŸ›‘οΈ **Exit** | Sets `uow_managed = False`. | Cleans up the session state for the next operation. | +* **Session Native:** The UOW operates on a standard SQLAlchemy `AsyncSession`. You can perform raw SQL executions within a UOW block, and they will be part of the same atomic transaction. +* **Service Interlock:** ZCore's `BaseService` is programmed to check for the `uow_managed` flag. It will only commit if it is the primary owner of the session, ensuring it doesn't break a UOW's boundary [service/base.py]. +* **Bypass:** For single-operation requests (like a simple GET or a standalone POST), you do not need the UOW. ZCore services will handle their own "Safe Commit" logic automatically. --- -## πŸ›‘οΈ Safe Post-Commit Event Dispatching - -A common engineering pitfall is dispatching external actionsβ€”like sending an SMS or notifying a payment gatewayβ€”before the database has actually saved the data. If the database save fails at the last millisecond, your system becomes inconsistent (e.g., a customer gets a "Thank You" email for an order that was never saved). - -ZCore solves this by buffering events. Events are only dispatched if `session.commit()` is successful: +## Under-the-Hood Spec -```python -async def commit(self) -> None: - try: - await self.session.commit() - except Exception as e: - logger.error(f"Transaction commit failed: {e}") - await self.session.rollback() - raise - - # Only reached if the database save was 100% successful - while self._pending_events: - event_name, payload = self._pending_events.pop(0) - try: - await self.dispatcher.dispatch(event_name, payload) - except Exception as ex: - # Failed handlers are logged but don't break the transaction - logger.error(f"Event handler failed for event '{event_name}': {ex}") -``` - ---- - -## πŸ’» Practical Usage - -Using the Unit of Work is straightforward. It is designed to wrap around your service calls: - -```python -async def process_checkout(self, user_id, cart_items): - # The UOW ensures both order creation and inventory update succeed together - async with UnitOfWork(self.session, self.dispatcher) as uow: - order = await self.order_service.create_order(user_id, cart_items) - await self.stock_service.reduce_inventory(cart_items) - - # Register an event that runs only after a successful save - uow.register_event("order.processed", {"order_id": order.id}) -``` - ---- +### 1. The `uow_managed` Metadata +When entering the `async with` block, the UOW sets `session.info["uow_managed"] = True` [db/uow.py]. This is a standard SQLAlchemy metadata dictionary. ZCore services inspect this flag before calling `commit()`. If `True`, they execute a `flush()` to send changes to the DB buffer but skip the `commit()`, delegating finality to the UOW. -## πŸ’‘ Engineering Insights +### 2. Transactional Event Buffering +The UOW maintains an internal `_pending_events` list [db/uow.py]. When `register_event()` is called, the event is not fired immediately. Instead, it is stored in the buffer. The UOW only iterates through this list and calls `dispatcher.dispatch()` **after** `session.commit()` has successfully returned. -!!! tip "πŸ’‘ Ensuring Atomicity" - The Unit of Work is your best tool for ensuring **Atomicity**. If your `reduce_inventory` call fails, the `order` created just a line above will be automatically rolled back from the database as if it never happened. +### 3. Automated Lifecycle Safety +The `__aexit__` method handles the logic of the transaction [db/uow.py]. If an exception was raised inside the block, it triggers `rollback()` and clears the event buffer. If no exception occurred, it triggers `commit()`. This ensures that even if a developer forgets to call commit manually, the transaction is handled correctly by the context manager. -!!! info "πŸ›‘οΈ Fault Isolation" - ZCore wraps every event handler in a `try/except` block. This ensures that if a "Welcome Email" service is temporarily down, it won't crash your entire checkout process or prevent other listeners from working. +!!! success "Event Isolation" + If a database commit succeeds but an event handler subsequently fails, the database remains committed. ZCore logs the event handler failure separately, ensuring that side effects (like failing to send an email) do not retroactively break your database integrity. \ No newline at end of file diff --git a/docs/features/ecosystem/cache.md b/docs/features/ecosystem/cache.md index 8c45db6..11dc3cf 100644 --- a/docs/features/ecosystem/cache.md +++ b/docs/features/ecosystem/cache.md @@ -1,101 +1,127 @@ -# ⚑ Distributed Caching & Local Eviction +# Distributed Caching -ZCore features a practical hybrid caching strategy. It utilizes a modest, thread-safe local cache (`TTLLRUCache`) as its primary layer, with the ability to scale to a distributed Redis cache when your environment requires it. This "two-tier" approach ensures that even if your central cache server goes offline, your application remains responsive. +Accelerate response times with a transparent, dual-layer caching system that routes through Redis and falls back to a thread-safe in-memory LRU store with automatic TTL eviction. --- -## πŸ“ Technical Architecture - -The caching system is designed to prioritize speed while maintaining a safety net. It automatically coordinates between in-memory storage and external distributed storage. - -```mermaid -flowchart TD - Get[Request Cache Key] --> CheckRedis{Redis Configured?} - CheckRedis -->|Yes| QueryRedis[Query Redis Server] - QueryRedis -->|Hit| Return[Return Data] - QueryRedis -->|Miss / Error| QueryLocal[Query Local TTLLRUCache] - CheckRedis -->|No| QueryLocal - - QueryLocal -->|Hit| Return - QueryLocal -->|Miss / Expired| ReturnNone[Return None] - - subgraph Garbage Collection [Background Worker] - Loop[Eviction Loop] -->|Every 60s| Sweep[Sweep Active Caches] - Sweep -->|Weak Reference| Purge[Purge Expired Keys] - end -``` +
+
+ Type
+ Performance Layer +
+
+ Status
+ Optional Ecosystem +
+
+ Underlying Tech
+ Redis / OrderedDict + threading +
+
+ +## The Challenge + +Caching in an async Python application introduces several subtle failure modes: + +1. **Cache Stampede Risk:** When the distributed cache (Redis) goes down, every request hits the database simultaneously, overwhelming the connection pool. +2. **Type Corruption:** Serialized JSON blobs are returned as raw dictionaries. Consumers must manually validate or cast the data back to the expected Pydantic model, often leading to `AttributeError` at runtime. +3. **Memory Leaks:** Naive in-memory caches keep references to objects indefinitely. In long-running async workers, this causes unbounded memory growth and eventual OOM kills. +4. **Stale Data Propagation:** Expired keys linger in memory until explicitly checked, returning stale data to users. + +## The ZCore Elegance + +ZCore provides `BaseCache`, a generic interface that transparently routes reads and writes through Redis when available, falling back to a local `TTLLRUCache` with thread-safe access, automatic TTL expiry, and weak-reference tracking for global eviction sweeps. + +=== "ZCore Dual-Layer Caching" + :::python + from zcore.cache import BaseCache, init_cache, close_cache + + # 1. Initialize Redis (optional) and eviction loop + await init_cache(redis_url="redis://localhost:6379/0") + + # 2. Create a typed cache for a domain + product_cache = BaseCache[ProductOut](prefix="products") + + # 3. Cache-aside pattern with automatic Pydantic validation + async def get_product(id: uuid.UUID) -> ProductOut: + cached = await product_cache.get(str(id), target_type=ProductOut) + if cached: + return cached + product = await fetch_from_db(id) + await product_cache.set(str(id), product.model_dump(), ttl=300) + return product + + # 4. Graceful shutdown + await close_cache() + +=== "Standard Manual Caching" + :::python + import json + import redis.asyncio as aioredis + from functools import lru_cache + + # Manual Redis client with no fallback + client = aioredis.from_url("redis://localhost:6379/0") + + @lru_cache(maxsize=128) # No TTL, no thread-safety in async + async def get_product(id: uuid.UUID) -> ProductOut: + raw = await client.get(f"product:{id}") + if raw: + data = json.loads(raw) + return ProductOut(**data) # No validation, no fallback on Redis failure + product = await fetch_from_db(id) + await client.set(f"product:{id}", product.model_dump_json(), ex=300) + return product + + # Redis failure β†’ all requests hit DB β†’ stampede + # lru_cache is not async-safe + # No global cleanup mechanism --- -## 🏒 1. Local Layer: The TTLLRUCache +## Cache Resolution Strategy -The local cache is a modest but robust in-memory store. It combines **Least Recently Used (LRU)** eviction with **Time-To-Live (TTL)** expiration to keep your memory footprint small. - -* **πŸ›‘οΈ Thread Safety**: Active read/write operations use reentrant locks (`threading.Lock()`). This prevents data corruption when multiple threads or tasks access the cache simultaneously. -* **πŸ“‰ LRU Eviction**: When the cache reaches its `maxsize` (default: 1000), it automatically discards the oldest, least-accessed items to make room for new ones. -* **🧹 Memory Leak Protection**: ZCore uses **Weak References** (`weakref.WeakSet`) to track active caches. This allows the Python garbage collector to remove cache instances from memory as soon as they are no longer needed, even if the background eviction loop is still running. +

+ Cache Resolution Strategy +

--- -## 🌍 2. Distributed Layer: BaseCache Abstraction - -The `BaseCache[T]` class provides a transparent interface. You don't need to write separate logic for Redis or local storage; the framework handles the "fallback" logic for you. +## Boundaries & Integration -### πŸ§ͺ Structured Data & Validation -When you retrieve data, ZCore doesn't just return a raw string. It automatically decodes the JSON and can validate it against a **Pydantic model**. This ensures that your cached data always matches the expected structure. +The caching layer is fully optional and designed to be composed with other ZCore components. -```python -# Internal validation logic -parsed_data = json_loads(raw_val) -if target_type: - return target_type.model_validate(parsed_data) -``` +* **Pydantic-First:** The `get` method accepts an optional `target_type` parameter. If a Pydantic model is provided, the deserialized dictionary is validated through `model_validate` before being returned. This guarantees type safety without manual casting. +* **Prefix Isolation:** Each `BaseCache` instance is created with a `prefix` string (e.g., `"products"`). All keys are automatically namespaced as `{prefix}:{key}`, preventing collision between different cache domains sharing the same Redis instance. +* **Background Eviction:** `init_cache` schedules a background `asyncio.Task` that runs every 60 seconds, calling `TTLLRUCache.evict_all_expired()` to purge stale records from all active in-memory cache instances. --- -## πŸ”„ Lifecycle Management +## Under-the-Hood Spec -ZCore manages the cache lifecycle through two modest functions typically called during application startup and shutdown. +### 1. BaseCache / TTLLRUCache Fallback Architecture -| Function | Responsibility | -| :--- | :--- | -| **`init_cache()`** | Connects to Redis and starts the background task that purges expired local keys every minute. | -| **`close_cache()`** | Stops the background worker and safely closes any open Redis connections. | - ---- +`BaseCache` holds a reference to a `TTLLRUCache` instance (default `maxsize=1000`) as `_local_cache` [cache/base.py]. On `set`, the method attempts Redis first. If Redis is unavailable or raises an exception (connection refused, timeout, etc.), the serialized value is written to the local LRU store. On `get`, the method tries Redis, then falls back to the local cache. -## πŸ’» Practical Usage +### 2. Pydantic Type Validation on Cache Hits -We suggest defining your caches within your domain modules. This keeps your keys organized and avoids naming collisions. +When `target_type` is provided and is a `BaseModel` subclass, the `get` method calls `target_type.model_validate(parsed_data)` [cache/base.py]. This ensures that the deserialized payload conforms exactly to the expected schema. If validation fails (e.g., due to schema drift between cache writes and reads), `None` is returned and the miss is treated as a cache miss. -```python -from zcore.cache.base import BaseCache -from pydantic import BaseModel +### 3. Eviction Loop Safety -# 1. Define what your cached data looks like -class ProductCache(BaseModel): - id: str - price: float +The global eviction loop `_start_eviction_loop` runs as a background `asyncio.Task` [cache/base.py]. It catches `asyncio.CancelledError` to exit cleanly and wraps the eviction call in a generic `except Exception` to prevent a single faulty cache instance from crashing the entire loop. -# 2. Create a namespaced cache instance -cache = BaseCache[ProductCache](prefix="products") +### 4. WeakRef Active Cache Registry -# 3. Set a value with a 1-hour expiration (TTL in seconds) -await cache.set("p_101", {"id": "p_101", "price": 29.99}, ttl=3600) - -# 4. Get the value back as a validated Pydantic object -product = await cache.get("p_101", target_type=ProductCache) -``` - ---- +`TTLLRUCache` registers every new instance in a module-level `weakref.WeakSet` called `_active_caches` [cache/ttllru_cache.py]. This enables `evict_all_expired` to iterate over all existing cache instances without preventing garbage collection of discarded instances. When an instance is garbage collected, its weak reference is automatically removed from the set. -## πŸ’‘ Engineering Insights +### 5. Thread-Safe LRU with Reentrant Lock -!!! tip "πŸ’‘ Graceful Fallback" - The caching system is designed for **Resilience**. If your Redis server encounters a network timeout or a crash, ZCore logs a warning and immediately tries to find the data in the local memory. Your users likely won't even notice the disruption. +Each `TTLLRUCache` instance uses a `threading.Lock` to protect all read and write operations [cache/ttllru_cache.py]. The underlying storage is an `OrderedDict` mapping keys to `(expiry_timestamp, value)` tuples. On `get`, expired keys are detected inline by comparing `time.time()` against the stored expiry, and are popped before `move_to_end` slides the remaining entry to the front. -!!! info "πŸ›‘οΈ Namespace Isolation" - Always use a `prefix` when initializing `BaseCache`. This ensures that a key named `101` in your "products" module doesn't overwrite a key named `101` in your "orders" module inside the shared Redis database. +!!! info "Redis Dependency" + The `BaseCache` Redis client (`redis.asyncio`) is imported lazily. If `redis` is not installed, `REDIS_AVAILABLE` is set to `False` and the cache operates exclusively in local fallback mode without raising import errors. -!!! warning "🧹 The Eviction Loop" - Remember to call `init_cache()` in your `main.py` or startup plugin. Without it, the background worker won't start, and expired local keys will remain in memory until the `maxsize` limit is reached. +!!! tip "Sizing the Local Cache" + The local `TTLLRUCache` defaults to a `maxsize` of 1000 entries. Adjust this in environments with high cardinality but limited memory. The LRU policy ensures that the most recently accessed items are always available. \ No newline at end of file diff --git a/docs/features/ecosystem/storage.md b/docs/features/ecosystem/storage.md index 656c35b..a88fd2f 100644 --- a/docs/features/ecosystem/storage.md +++ b/docs/features/ecosystem/storage.md @@ -1,110 +1,139 @@ -# πŸ“ File Storage & Security Validators +# File Storage & Security -ZCore provides a modest and reliable file storage system managed by the `StorageProvider` interface. It is designed to handle file uploads, raw binary streaming, and secure deletions while enforcing strict practical safeguards to protect your host system from malicious activities. +Upload, stream, and delete files through a pluggable provider interface with multi-layer validationβ€”extension checks, size limits, and magic byte MIME verificationβ€”all backed by path traversal protection. --- -## 🏠 Local Storage Provider (`LocalStorageProvider`) +
+
+ Type
+ File Management +
+
+ Status
+ Optional Ecosystem +
+
+ Underlying Tech
+ aiofiles / Magic Bytes +
+
+ +## The Challenge + +File upload handling in web applications is a frequent source of security vulnerabilities and operational headaches: + +1. **Unrestricted File Size:** Large uploads can exhaust disk space, memory, or worker I/O, leading to Denial of Service (DoS). +2. **Extension Masquerading:** An attacker uploads a `.png` file that contains a PHP web shell or a Windows executable. The server trusts the extension and executes the payload. +3. **Path Traversal:** Malicious filenames like `../../etc/passwd` trick naive storage code into writing outside the designated upload directory. +4. **Filename Collisions:** Multiple users uploading `avatar.jpg` overwrite each other's files. + +## The ZCore Elegance + +ZCore provides a `LocalStorageProvider` that enforces strict path bounds, generates collision-resistant filenames using truncated UUIDs, and chains configurable validators for extension, file size, and magic byte MIME verification. + +=== "ZCore Secure File Upload" + :::python + from zcore.storage import ( + LocalStorageProvider, + FileExtensionValidator, + MaxFileSizeValidator, + SafeMimeTypeValidator, + ) + + # 1. Configure layered validators + validators = [ + FileExtensionValidator(allowed_extensions=["png", "jpg", "pdf"]), + MaxFileSizeValidator(max_size_mb=10), + SafeMimeTypeValidator(allowed_mimes=["image/png", "image/jpeg", "application/pdf"]), + ] + + # 2. Create the provider + storage = LocalStorageProvider( + base_path="./uploads", + validators=validators, + ) + + # 3. Upload (validated and sanitized) + path = await storage.upload(file=uploaded_file, related_type="avatars") + # path β†’ "uploads/avatars/a1b2c3d4e5f6789.jpg" + +=== "Standard File Upload" + :::python + import aiofiles + import uuid + from pathlib import Path + + @app.post("/upload") + async def upload(file: UploadFile): + # Manual validationβ€”easy to skip or forget + if not file.filename.endswith((".png", ".jpg")): + raise HTTPException(400, "Invalid extension") + + # Vulnerable to path traversal + path = f"./uploads/{file.filename}" + + # No size limit enforcement + async with aiofiles.open(path, "wb") as f: + while chunk := await file.read(): + await f.write(chunk) + + # Filename collisions overwrite existing files -The `LocalStorageProvider` is the default implementation for managing files on your server's filesystem. It focuses on two core engineering principles: **Isolation** and **Predictability**. +--- -### πŸ›‘οΈ Traversal Exploit Mitigation -A common security risk is "Path Traversal," where a user provides a filename like `../../etc/passwd` to access sensitive system files. ZCore mitigates this by resolving every path to its absolute physical location and verifying it remains within your configured `base_path`. +## Validation Pipeline -```python -# Internal safety check -base = await Path(self.base_path).resolve(strict=False) -target_folder = await (Path(self.base_path) / folder_name).resolve(strict=False) +

+ Validation Pipeline +

-if not target_folder.is_relative_to(base): - raise AppException("Path traversal attempt detected") -``` -### πŸ”’ Collision-Resistant Filenames -To prevent users from overwriting each other's files and to hide original filenames (which might contain sensitive information), ZCore renames every upload using a randomized 15-character UUID while preserving the original extension. +--- -| Original Filename | Generated ZCore Path | -| :--- | :--- | -| `vacation_photo.jpg` | `uploads/images/a1b2c3d4e5f6g7h.jpg` | -| `report_2023.pdf` | `uploads/docs/z9y8x7w6v5u4t3s.pdf` | +## Boundaries & Integration ---- +The storage layer is designed for flexibility across deployment environments. -## 🚦 Upload Validation Pipeline - -Every file passing through the storage system must clear a series of "security gates." If a file fails any gate, the process is aborted, and no data is written to your disk. - -```mermaid -flowchart TD - Upload[Incoming File] --> Ext[1. Extension Gate] - Ext -->|Valid| Size[2. Size Gate] - Size -->|Under Limit| Magic[3. Content Gate] - - subgraph Content Gate [Deep Inspection] - Magic -->|Check Magic Bytes| Signature{Signature Match?} - Signature -->|Yes| ScriptCheck{Script Injection?} - ScriptCheck -->|Contains |Clean| Success[βœ… Safe to Save] - end - - Block --> Reject[Raise ValidationError] -``` - -### 1. FileExtensionValidator -This gate performs a case-insensitive check. Whether a user uploads `image.PNG` or `image.png`, the validator normalizes the input to ensure it matches your allowed list. - -### 2. MaxFileSizeValidator -To defend against **Denial of Service (DoS)** attacks where a user attempts to fill your hard drive with massive files, this validator enforces a strict size limit. It uses an efficient "seek" method to determine file size without loading the entire file into memory. - -### 3. SafeMimeTypeValidator (The "Magic Byte" Check) -Simply checking a file extension is not enough, as a user could rename a malicious script to `photo.jpg`. ZCore inspects the first **2048 bytes** of the file to verify its "Magic Bytes"β€”the unique digital signature that identifies true file types (e.g., PNGs always start with specific bytes). - -#### πŸ›‘οΈ Malicious Pattern Blocking -The validator scans the file header for high-risk patterns. ZCore will automatically block and log any file containing: -* 🐘 **PHP Tags:** ` CheckRedis{Redis Active?} - CheckRedis -->|Yes| RedisPub[Broadcast to Redis
'stream:user:id'] - CheckRedis -->|No| LocalPub[_local_publish] - - RedisPub -->|Nodes Listen| RedisListen[Incoming Redis Message] - RedisListen --> LocalPub - - subgraph Node [Local Server Node] - LocalPub -->|Find Queues| Loop[Push to asyncio.Queue] - Loop -->|Queue Full| Evict[🚨 Drop Stalled Queue] - Loop -->|Success| Client[Client Receives Event] - end -``` +
+
+ Type
+ Real-Time Messaging +
+
+ Status
+ Optional Ecosystem +
+
+ Underlying Tech
+ Redis PubSub / asyncio.Queue +
+
+ +## The Challenge + +Building real-time user-targeted event streaming in a horizontally-scaled web application introduces several hard problems: + +1. **Multi-Node Broadcast:** In a standard setup, a WebSocket connection is pinned to a single server process. An event generated on Node A cannot reach a user connected to Node B without a shared backplane. +2. **Queue Overrun:** Naive in-process queues grow unboundedly when a slow consumer cannot keep up, eventually exhausting memory and crashing the worker. +3. **Subscription Leaks:** When a WebSocket disconnects, the associated event listener must be cleaned up. Failing to do so results in ghost subscriptions that keep processing and queuing events for connections that no longer exist. + +## The ZCore Elegance + +The `StreamManager` provides a subscription-based event streaming system. It maintains a per-user mapping of bounded `asyncio.Queue` instances, routes published events through Redis PubSub to broadcast across all application nodes, and falls back to local in-memory delivery when Redis is unavailable. + +=== "ZCore Real-Time Streaming" + :::python + from zcore.web.streams import StreamManager + from zcore.web.api_router import ZCoreAPIRoute + + stream_manager = StreamManager() + + @app.websocket("/ws/events") + async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + user_id = get_authenticated_user_id(websocket) + + # 1. Subscribe via safe context manager + async with stream_manager.subscription(user_id) as queue: + while True: + try: + event = await asyncio.wait_for(queue.get(), timeout=30) + await websocket.send_json(event) + except asyncio.TimeoutError: + await websocket.send_json({"type": "heartbeat"}) + + # Publish from any service (any node) + await stream_manager.publish(user_id, {"type": "order_update", "order_id": "123"}) + +=== "Standard WebSocket Handling" + :::python + # Single-node only, no cluster support + active_connections: dict[uuid.UUID, list[WebSocket]] = {} + + @app.websocket("/ws") + async def ws(websocket: WebSocket): + user_id = extract_user(websocket) + active_connections.setdefault(user_id, []).append(websocket) + try: + while True: + data = await websocket.receive_text() + # No Redis backplane β€” cannot reach other nodes + except WebSocketDisconnect: + # Manual cleanupβ€”easy to forget or mis-handle + if user_id in active_connections: + active_connections[user_id].remove(websocket) --- -## 1. Local Queue & Connection Scoping +## Streaming Architecture -When a client (like a browser or mobile app) wants to receive live updates, ZCore creates a private "mailbox" for that connection using an `asyncio.Queue`. +

+ Streaming Architecture +

-### πŸ›‘οΈ Context-Safe Subscriptions -To prevent "ghost connections" and memory leaks, ZCore uses an asynchronous context manager. This guarantees that as soon as a user disconnects or the request ends, their queue is automatically removed and cleaned up. - -```python -# Usage inside a route -async with stream_manager.subscription(user_id) as queue: - # While the connection is open, wait for new mail - data = await queue.get() - # Send data to client... -# Once the block exits, the queue is destroyed automatically. -``` --- -## 2. Scaling Across Multiple Nodes - -In a production environment with multiple servers, a user might be connected to **Server A**, but the event that triggers their notification happens on **Server B**. +## Boundaries & Integration -ZCore solves this using **Redis PubSub**. When an event is published, it is broadcast to a specific channel (`stream:user:`). Every server in your cluster listens to these channels. When **Server A** hears the message from Redis, it checks if that user is connected locally and delivers the message to their queue. +The streaming layer is designed for horizontal scalability and clean lifecycle management. -| Feature | Single Node | Multi-Node (Redis) | -| :--- | :--- | :--- | -| **Delivery** | Direct memory transfer. | Broadcast via Redis. | -| **Latency** | Near-zero. | Minimal (Network hop). | -| **Complexity** | Very Low. | Modest (Requires Redis). | +* **Redis PubSub Fallback:** The `publish` method attempts Redis first. If Redis is not configured or the call fails (timeout, connection refused), the event is routed locally via `_local_publish`. This ensures that streaming works in development without Redis and degrades gracefully in production during Redis outages. +* **Dependency Injection Friendly:** `StreamManager` is a plain Python class that can be registered as a singleton in the IoC container. Its `publish` method can be injected into any service or background task without coupling to the Web layer. +* **WebSocket Agnostic:** The `StreamManager` has no dependency on `WebSocket` or `Starlette`. It manages `asyncio.Queue` objects keyed by `uuid.UUID`. This allows it to be used with custom protocols, SSE (Server-Sent Events), or long-polling implementations. --- -## 3. πŸ›‘οΈ Memory & Overflow Protection +## Under-the-Hood Spec -Real-time systems can be vulnerable if a client connection "stalls" (stops reading data but stays connected). If the server keeps pushing events into a stalled queue, it will eventually run out of memory. +### 1. Redis PubSub Backplane with Pattern Subscription -ZCore protects your server by enforcing a **Max Size** on every queue. If a queue reaches 100 pending events and the client still hasn't read them, ZCore modestly assumes the connection is dead or unhealthy and drops that specific queue to protect system resources. +The `StreamManager` subscribes to the Redis pattern `stream:user:*` using `psubscribe` [web/streams.py]. This is a single subscription that matches all user-specific channels. When a message arrives, the handler parses the user ID from the channel name (`channel.split(":")[-1]`) and forwards the JSON-deserialized payload to the matching local queues. -```python -# Internal safety logic -try: - queue.put_nowait(data) # Try to deliver instantly -except asyncio.QueueFull: - # If the client is too slow, remove them to save memory - self.users_queues[user_id].remove(queue) -``` - ---- +### 2. Bounded Queue Overflow Protection -## πŸ’» Practical Usage (SSE Example) +Each user subscription creates an `asyncio.Queue(maxsize=100)` [web/streams.py]. When `_local_publish` distributes an event, it calls `queue.put_nowait(data)`. If the queue is full, `put_nowait` raises `asyncio.QueueFull`. The handler catches this and **removes the overflowing queue** from the active list, effectively disconnecting the slowest consumer for that user. This prevents one slow WebSocket from causing memory exhaustion for the entire process. -We suggest using **Server-Sent Events (SSE)** for simple one-way real-time updates. It is lightweight and works natively in modern browsers. +### 3. Subscription Context Manager with Guaranteed Cleanup -```python -import uuid -from fastapi import APIRouter -from fastapi.responses import StreamingResponse -from zcore.web.streams import StreamManager +The `subscription(user_id)` method is an `@asynccontextmanager` [web/streams.py]. It calls `subscribe` on entry (which creates the queue and starts the Redis PubSub listener if needed) and `unsubscribe` on exit (which removes the queue from the user's list and cancels the PubSub background task if no more active queues exist). This pattern guarantees cleanup even if the WebSocket handler raises an unhandled exception. -router = APIRouter() -stream_manager = StreamManager() +### 4. Background Listener Lifecycle -@router.get("/notifications/{user_id}") -async def stream_notifications(user_id: uuid.UUID): - async def event_generator(): - # 1. Open the safe subscription context - async with stream_manager.subscription(user_id) as queue: - while True: - # 2. Wait for the next event - data = await queue.get() - # 3. Format as an SSE message - yield f"data: {data}\n\n" - - return StreamingResponse(event_generator(), media_type="text/event-stream") -``` - ---- +`start_listening` initializes the Redis PubSub connection and spawns `_listen_to_redis` as a background `asyncio.Task` [web/streams.py]. The task runs until cancelled. In the `finally` block, it always calls `punsubscribe` and `close()` on the PubSub connection, ensuring Redis resources are released. The task is lazily started on the first `subscribe` call and automatically cancelled in `unsubscribe` when the last queue is removed. -## πŸ’‘ Engineering Insights +### 5. Thread-Safe Queue State with Async Lock -!!! tip "πŸ’‘ Redis is Optional" - If you are running a small application on a single server, you don't need Redis. The `StreamManager` will automatically detect that no Redis client is available and route everything through local memory. +All mutations to the `users_queues` dictionary are protected by `self._lock`, an `asyncio.Lock` [web/streams.py]. This prevents race conditions where a concurrent `publish` and `unsubscribe` could modify the same user's queue list simultaneously in an async context. -!!! info "πŸ›‘οΈ Namespacing" - Events are scoped by `user_id`. This ensures that User A never receives a private message intended for User B. If you need to send "Global" events, you can use a fixed system UUID as the `user_id` for a public broadcast channel. +!!! info "Redis Client Isolation" + The streaming subsystem maintains its own module-level `_stream_redis_client`, separate from the caching layer's `_shared_redis_client`. Each subsystem manages its own connection lifecycle and configuration. -!!! warning "🧹 Background Tasks" - The background Redis listener starts automatically when the first user connects and shuts down when the last user disconnects. This "lazy" management ensures that no background resources are wasted when no one is listening. +!!! tip "Horizontal Scaling" + To scale WebSocket streams across multiple application instances, ensure all instances share the same Redis instance. Each node subscribes to the same `stream:user:*` pattern via its own PubSub connection, guaranteeing that an event published on any node reaches the user regardless of which node they are connected to. \ No newline at end of file diff --git a/docs/features/infrastructure/cli.md b/docs/features/infrastructure/cli.md index 62bdec6..2ebecba 100644 --- a/docs/features/infrastructure/cli.md +++ b/docs/features/infrastructure/cli.md @@ -1,83 +1,77 @@ -# πŸ› οΈ Command-Line Scaffolding Tool (CLI) +# CLI Scaffolding & Orchestration -The `zc` command-line utility is a modest but practical tool designed to help you maintain a consistent project structure. It automates the repetitive task of creating files and folders, ensuring that every module in your system follows the same architectural blueprint. +Standardize your project structure and automate repetitive boilerplate generation with a professional command-line interface. --- -## πŸ“‹ Core Scaffolding Commands - -The `zc` CLI supports four primary operations to help you move from an idea to a running service in minutes. - -| Command | Usage | Resulting Files | Purpose | -| :--- | :--- | :--- | :--- | -| ✨ **`init`** | `zc init ` | `main.py`, `.env`, `requirements.txt` | Initializes a new project root. | -| πŸš€ **`startapp`** | `zc startapp ` | `models.py`, `services.py`, `plugin.py`, etc. | Scaffolds a modular domain plugin. | -| πŸ”₯ **`run`** | `zc run` | *(Uvicorn Process)* | Launches the local development server. | -| πŸ”‘ **`gensecret`** | `zc gensecret` | `64-char-hex-string` | Generates a secure `SECRET_KEY`. | +
+
+ Type
+ Management Utility +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ Pathlib / secrets / subprocess +
+
+ +## The Challenge +FastAPI is intentionally minimalist, which means it lacks a built-in project generator or application "scaffolder" like Django's `startapp`. This leads to: +1. **Structural Inconsistency:** Different projects within the same team follow different folder patterns. +2. **Boilerplate Fatigue:** Developers spend hours manually creating `models.py`, `schemas.py`, `repositories.py`, and `services.py` for every new domain. +3. **Pathing Issues:** Difficulty managing `PYTHONPATH` so that modular sub-apps can import each other without complex hacks. +4. **Insecure Defaults:** Developers often use weak, hardcoded secret keys during initial setup. + +## The ZCore Elegance +The `zc` CLI tool provides a unified command set to initialize projects, scaffold modular domain-driven apps, and launch a pre-configured development server. It enforces the ZCore architectural standards while generating cryptographically secure environment files. + +=== "ZCore CLI Orchestration" + :::bash + # 1. Initialize a new project with secure secrets + zc init my_api + cd my_api + + # 2. Scaffold a domain with full Repository/Service logic + zc startapp inventory --template + + # 3. Launch the server with correct PYTHONPATH + zc run + +=== "Standard Manual Scaffolding" + :::bash + # Standard projects require manual labor for every module: + mkdir -p inventory + touch inventory/__init__.py inventory/models.py inventory/services.py + # (Open editor, manually write 100+ lines of CRUD boilerplate) + # (Manually manage .env and secret generation) + # (Manually run uvicorn with path overrides) + PYTHONPATH=. uvicorn main:app --reload --- -## πŸ“ Scaffolding Architecture - -The CLI is engineered to prevent common naming mistakes. It ensures that your files are valid Python modules and your database tables follow clean `snake_case` naming conventions. - -```mermaid -flowchart TD - CLI[zc startapp product_catalog] -->|Identifier Check| Check{Is Name Valid?} - Check -->|No| Reject[🚨 Abort & Explain Why] - Check -->|Yes| Format[Convert snake_case to PascalCase] - Format -->|Map Templates| Template[ProductCatalog Model & Tables] - Template -->|Write to Disk| Disk[Generate products_catalog/ folder] -``` - -### πŸ›‘οΈ Guardrails for Consistency -Before creating any files, the CLI verifies your input against Python's naming rules. This prevents issues where an app name might conflict with Python keywords or contain invalid characters. - ---- - -## πŸ“ Scaffolding Options - -ZCore offers two ways to generate your application modules depending on how much control you want from the start: +## Boundaries & Integration +The CLI is a developer productivity tool, not a runtime dependency. -1. **Clean Slate:** Running `zc startapp products` creates the necessary files but keeps them empty, allowing you to define your logic from a blank canvas. -2. **Guided Template:** Running `zc startapp products -t` (or `--template`) pre-populates the files with working ZCore boilerplates. This includes a sample Model, Schema, Repository, Service, and Router already wired together. +* **Standard Python Output:** Every file generated by `zc` is a standard `.py` file based on the templates in `zcore.cli.templates`. You can delete the CLI and your project will continue to function perfectly. +* **Uvicorn Wrapper:** The `zc run` command is a wrapper around `subprocess.run` calling `uvicorn`. It reads the `HOST` and `PORT` directly from your `.env` file to ensure the CLI environment matches your application settings [cli/commands.py]. +* **Plugin Registration:** After running `startapp`, ZCore reminds you to register the generated `Plugin` in your `Kernel`. The CLI does not "magically" modify your `main.py` code to prevent accidental corruption of your project logic. --- -## πŸ’» Practical Usage Guide - -### 1. Initialize a Project -When you run `init`, ZCore automatically generates a `.env` file pre-configured with a unique, cryptographically secure secret key. - -```bash -zc init my_new_api -cd my_new_api -``` - -### 2. Scaffold a New Domain App -We recommend using the `-t` flag for your first few modules to see how the layers are intended to interact. - -```bash -# Generates a structured module with boilerplate code -zc startapp order_management -t -``` - -### 3. Launch the Development Server -The `run` command is a modest wrapper around Uvicorn. It automatically looks for your `.env` file to set the correct host and port. - -```bash -zc run -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. Cryptographically Secure Secret Generation +The `zc init` and `zc gensecret` commands use the `secrets` module (specifically `secrets.token_hex(32)`) [cli/commands.py]. This ensures that every ZCore project starts with a unique, high-entropy 64-character secret key, mitigating common production vulnerabilities caused by weak default keys. -!!! tip "πŸ’‘ Why use the CLI?" - Consistency is the key to maintainability. By using the CLI, you ensure that every developer on your team starts with the same file names and the same structural patterns, making it much easier to navigate the codebase as it grows. +### 2. Dynamic PYTHONPATH Injection +Modular applications often fail to resolve imports when running via raw `python` or `uvicorn`. The `zc run` command dynamically modifies the execution environment by setting `env["PYTHONPATH"] = "."` [cli/commands.py]. This ensures that your modular plugins can use clean, absolute imports (e.g., `from inventory.models import Stock`) regardless of the current working directory. -!!! info "πŸ”‘ Secret Key Generation" - The `gensecret` command uses Python's standard `secrets` module to generate a 256-bit token. You should use this to replace the default `SECRET_KEY` in your production environment to ensure your JWT tokens are securely signed. +### 3. Identifier & Path Validation +Before creating any files, the CLI validates that the project and app names are valid Python identifiers using `name.isidentifier()` [cli/main.py]. It also utilizes `pathlib` for strict collision checks, preventing the CLI from overwriting existing files if a developer runs a scaffolding command by mistake. -!!! warning "πŸ›‘οΈ Valid Identifiers" - If you try to name an app `123-app` or `my app`, the CLI will block the operation. Python modules must start with a letter and contain only alphanumeric characters or underscores. +!!! info "CLI Help" + Run `zc --help` to see all available commands, including `gensecret` for rotating keys and version checks. \ No newline at end of file diff --git a/docs/features/infrastructure/config.md b/docs/features/infrastructure/config.md index fe6c33c..d1cb48b 100644 --- a/docs/features/infrastructure/config.md +++ b/docs/features/infrastructure/config.md @@ -1,99 +1,74 @@ -# βš™οΈ Configuration Management +# Centralized Configuration Management -ZCore manages application settings using a modest and practical approach based on **Pydantic Settings (v2)**. This ensures that your configurations are validated, type-safe, and driven by environment variables, while remaining easy to manage as your project grows. +Type-safe, environment-aware settings with lazy resolution and seamless Dependency Injection integration. --- -## πŸ“‹ Configuration Schema (`ZCoreCoreSettings`) - -Core settings are defined in a central schema. This class automatically parses, validates, and converts values from your system's environment variables or a `.env` file at runtime. - -| Variable | Default Value | Purpose | -| :--- | :--- | :--- | -| 🌍 `ENVIRONMENT` | `"production"` | Context: `production`, `development`, or `test`. | -| πŸ“› `PROJECT_NAME` | `"ZCore App"` | Branding used in OpenAPI/Swagger metadata. | -| πŸ—„οΈ `DATABASE_URL` | `sqlite+aiosqlite...` | Primary SQL connection string. | -| πŸ§ͺ `DATABASE_TEST` | `sqlite+aiosqlite...` | Isolated database used during automated tests. | -| 🏊 `POOL_SIZE` | `5` | Connection pool size (bypassed for SQLite). | -| πŸ”‘ `SECRET_KEY` | *(Insecure Fallback)* | Cryptographic key for signing JWTs and hashes. | -| ⏳ `ACCESS_TOKEN...` | `30` | Access token lifespan in minutes. | -| πŸ“ `STORAGE_PATH` | `"./storage"` | Local directory reserved for file uploads. | -| πŸ›°οΈ `REDIS_URL` | `None` | Redis connection URI (if caching is active). | - ---- - -## πŸ“‚ Environmental File Loading - -The configuration manager looks for a `.env` file in your project root by default. You can easily switch between different configuration files (e.g., for testing) by setting the `ENV_FILE` environment variable. - -```bash -# Example: Run tests using a specific test environment file -ENV_FILE=.env.test pytest tests/ -``` +
+
+ Type
+ Configuration Utility +
+
+ Status
+ Core Infrastructure +
+
+ Underlying Tech
+ Pydantic Settings V2 +
+
+ +## The Challenge +Managing configuration in growing FastAPI projects often results in **Circular Import Hell**. This happens because nearly every module (Database, Security, Services) needs settings, but settings are often instantiated in `main.py`. If a module imported by `main.py` tries to import the settings instance from `main.py`, the app crashes. + +Furthermore, manually managing `.env` file paths and ensuring type-safety for environment variables across different deployment stages (dev, test, prod) adds unnecessary cognitive load. + +## The ZCore Elegance +ZCore solves this using a **Settings Proxy** combined with the global **IoC Container**. You simply import a global `settings` object. It remains "hollow" until an attribute is accessed, at which point it dynamically resolves the configuration from the container. This eliminates circular imports and allows for easy settings overriding during testing. + +=== "ZCore Lazy Configuration" + :::python + # Import and use anywhere - zero circular import risk + from zcore import settings + + def get_db_url(): + # Instance is resolved lazily on first access + return settings.DATABASE_URL + +=== "FastAPI Manual Management" + :::python + # Standard approach often leads to circularity or manual passing + from my_app.main import settings # Risk: main imports modules, modules import main + + # OR: Manual instantiation in every file (Expensive/Inconsistent) + from pydantic_settings import BaseSettings + class MySettings(BaseSettings): + DATABASE_URL: str + + settings = MySettings() # Re-reading .env every time --- -## 🧠 Lazy Resolution via `SettingsProxy` - -A common issue in large applications is "circular imports" or slow startups caused by loading configurations too early. ZCore solves this through a **Lazy-Resolution Proxy**. - -```mermaid -sequenceDiagram - participant Dev as Developer Code - participant Proxy as SettingsProxy - participant IoC as IoC Container - participant Inst as Settings Instance - - Dev->>Proxy: Import settings - Note over Proxy: (No work done yet) - Dev->>Proxy: Access .DATABASE_URL - Proxy->>IoC: Check for existing settings - IoC-->>Proxy: Not found - Proxy->>Inst: Create & Register Singleton - Inst-->>Proxy: Return Value - Proxy-->>Dev: "sqlite+aiosqlite:///..." -``` +## Boundaries & Integration +ZCore Configuration is a transparent wrapper around the Pydantic ecosystem. -!!! info "πŸ›‘οΈ Engineered for Efficiency" - The `settings` object you import is actually a "Proxy." It remains dormant until you access an attribute for the first time. At that moment, it communicates with the **IoC Container** to instantiate the settings as a global singleton. +* **Pydantic V2 Native:** `ZCoreCoreSettings` inherits directly from `pydantic_settings.BaseSettings`. You can use all Pydantic features like `AliasPath`, `Field` validation, and complex types (SecretStr, HttpUrl) [config.py]. +* **Extensible:** You can easily extend the base settings by creating your own class. Once registered via `initialize_settings(my_inst)`, ZCore's global `settings` proxy will resolve to your custom implementation. +* **Bypass:** If you prefer not to use the global proxy, you can instantiate your own Pydantic Settings class and use it normally. ZCore's internal modules will continue to use their registered singletons without interference. --- -## πŸ’» Practical Usage - -We suggest importing the global `settings` proxy to access your configurations anywhere in your application. - -```python -from zcore.config import settings - -# Safe, lazy access to your environment variables -db_url = settings.DATABASE_URL -is_dev = settings.ENVIRONMENT == "development" -``` - -### πŸ› οΈ Overriding Settings for Testing -If you need to change settings dynamically (for example, using an in-memory database during unit tests), you can register a custom subclass in the IoC container before the application boots: - -```python -from zcore.config import ZCoreCoreSettings, initialize_settings - -class TestSettings(ZCoreCoreSettings): - DATABASE_URL: str = "sqlite+aiosqlite:///:memory:" - ENVIRONMENT: str = "test" - -# Register this as the global settings instance -initialize_settings(TestSettings()) -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The Settings Proxy +The global `settings` object is an instance of `SettingsProxy` [config.py]. It implements `__getattr__`, which intercepts attribute access and calls `get_settings()`. This function attempts to resolve the settings instance from the `IoCContainer`. If not found, it instantiates the default `ZCoreCoreSettings`, registers it as a singleton, and returns it. -!!! tip "πŸ’‘ Type Safety" - Because ZCore uses Pydantic, your settings are strictly typed. If you define `POOL_SIZE: int` but provide `"abc"` in your `.env` file, ZCore will raise a clear validation error during startup, preventing your app from running in an unstable state. +### 2. Dynamic `.env` Resolution +The `model_config` is configured to look for an environment variable named `ENV_FILE` [config.py]. If set (e.g., `ENV_FILE=.env.test`), ZCore will load that specific file. If not set, it defaults to the standard `.env` in the project root. This allows for seamless switching between environment configurations without code changes. -!!! warning "πŸ›‘οΈ Production Security" - In a production environment, never rely on the default `SECRET_KEY`. ZCore will log a warning (or abort in strict mode) if it detects the insecure fallback key is active. Use `zc gensecret` to create a strong, unique key for your deployment. +### 3. DI Container Registration +The `initialize_settings` function performs a "Double Registration" [config.py]. It registers the settings instance under its specific class type *and* under the base `ZCoreCoreSettings` type. This ensures that even if you provide a custom subclass (e.g., `MyAppSettings`), ZCore's internal components can still inject the base type and receive your custom instance. -!!! info "πŸ§ͺ Extra Variables" - The `ZCoreCoreSettings` class is configured to ignore "extra" variables in your `.env` file. This allows you to keep other configuration data in the same file without causing Pydantic validation errors. +!!! info "Production Security" + ZCore's settings default to `ENVIRONMENT="production"`. In this mode, certain security-critical defaults (like the insecure `SECRET_KEY`) will trigger fatal errors in related modules like `zcore.security.jwt` to prevent accidental insecure deployments. \ No newline at end of file diff --git a/docs/features/infrastructure/di.md b/docs/features/infrastructure/di.md index 1642daa..04de179 100644 --- a/docs/features/infrastructure/di.md +++ b/docs/features/infrastructure/di.md @@ -1,99 +1,119 @@ -# πŸ’‰ Dependency Injection (IoC Container) +# Scoped Dependency Injection -Dependency Injection (DI) is a modest but powerful pattern that allows your application to remain decoupled and easy to test. Instead of a class creating its own "helpers," the framework "injects" them. ZCore’s built-in Inversion of Control (IoC) container manages these dependencies for you, handling everything from simple utilities to complex database sessions. +Manage object lifecycles and resolve complex dependency graphs through thread-safe, constructor-based injection. --- -## ⏳ Lifecycle Management - -The ZCore `IoCContainer` manages objects across three practical lifecycles. Choosing the right lifecycle ensures your application uses memory efficiently and keeps data isolated where necessary. - -| Lifecycle | Scope | Practical Use Case | -| :--- | :--- | :--- | -| 🌍 **Singleton** | Shared globally across the entire application. | Event dispatchers, config managers, database engines. | -| πŸ”„ **Scoped** | Created once per HTTP request and then destroyed. | Database sessions, current user context, repositories. | -| ✨ **Transient** | A fresh instance is created every time it's requested. | Calculation helpers, data formatters, short-lived tasks. | - ---- - -## πŸ“ Technical Architecture & Isolation - -One of the most important jobs of a DI container in a web framework is ensuring that **User A's data never leaks into User B's request**. ZCore uses "Context Variables" (`contextvars`) to create a thread-safe sandbox for every request. - -```mermaid -flowchart TD - Request[Incoming HTTP Request] -->|Middleware Assigns| ScopeID[Unique Scope ID] - ScopeID -->|Binds to| ContextVar[_current_scope_id] - ContextVar -->|Resolves Scoped Binding| ContextInstance[_scoped_instances] - ContextInstance -->|Shared within this Request| Repo[Repositories / Services] - Request -->|Request Finished| Purge[clear_scope] - Purge -->|Clean up memory| ContextInstance -``` - -* **`_current_scope_id`**: A tracking ID for the active request. -* **`_scoped_instances`**: A private dictionary for the current request. When the request ends, ZCore modestly clears this dictionary to prevent memory pollution. +
+
+ Type
+ Inversion of Control (IoC) +
+
+ Status
+ Core Infrastructure +
+
+ Underlying Tech
+ contextvars / Reflection +
+
+ +## The Challenge +In standard FastAPI, dependencies are primarily resolved at the route level via `Depends()`. While powerful, this leads to: +1. **Signature Bloat:** Passing the same 5 dependencies through every function layer (Router -> Service -> Repository). +2. **Lifecycle Confusion:** Difficulty maintaining "Request-Scoped" objects (like a database session) consistently across multiple nested services. +3. **Tight Coupling:** Classes manually instantiating their dependencies, making units hard to mock during testing. +4. **Circular References:** No built-in protection against recursive dependency loops during instantiation. + +## The ZCore Elegance +ZCore provides a central `IoCContainer` that supports **Singleton**, **Scoped**, and **Transient** lifecycles. It uses constructor-based injectionβ€”meaning your services and repositories simply declare their needs in `__init__`. ZCore analyzes the signatures, resolves the graph, and detects circular dependencies before your code even executes. + +=== "ZCore Constructor Injection" + :::python + from zcore import Inject + + class PaymentService: + # Dependencies are declared in the constructor + def __init__( + self, + repo: PaymentRepository, + notify: EmailService + ): + self.repo = repo + self.notify = notify + + # In the Router: Zero manual wiring + @router.post("/") + async def pay(service: PaymentService = Inject(PaymentService)): + return await service.process() + +=== "FastAPI Raw Implementation" + :::python + # Dependencies must be manually passed through every layer + @router.post("/") + async def pay( + db: AsyncSession = Depends(get_db), + email_client: EmailClient = Depends(get_email_client) + ): + # Manual instantiation or repetitive Depends() + repo = PaymentRepository(db) + notify = EmailService(email_client) + service = PaymentService(repo, notify) + return await service.process() + --- -## 🧠 High-Performance Reflection Caching - -Inspecting Python code at runtime (Reflection) can be slow. To ensure ZCore remains responsive, the auto-wiring engine uses a two-stage caching strategy: +## Resolution Workflow +The IoC container follows a strict resolution path, checking lifecycles in order of persistence. -1. **Constructor Caching**: We cache the `__init__` method of your classes so we don't have to look them up repeatedly. -2. **Signature Caching**: We resolve type hints (like `repository: ProductRepository`) once and store them. Subsequent requests skip the "discovery" phase and jump straight to "instantiation." +

+ Resolution Workflow +

--- -## πŸ›‘οΈ Safety: Circular Dependency Detection +## Boundaries & Integration +ZCore's DI system is a bridge between standard Python classes and the FastAPI ecosystem. -In complex systems, it's easy to accidentally create a loop (e.g., `ServiceA` needs `ServiceB`, but `ServiceB` needs `ServiceA`). Without protection, this would cause the application to crash or freeze. - -ZCore implements a recursive tracking stack. If it detects that it is trying to resolve the same class twice in a single "branch" of the dependency tree, it raises a `CircularDependencyError` with a clear map of the loop: - -```text -❌ CircularDependencyError: Circular dependency detected: - ServiceA -> ServiceB -> ServiceA -``` +* **FastAPI `Depends` Wrapper:** The `Inject()` function is a lightweight wrapper. It returns a FastAPI `Depends` object, ensuring full compatibility with FastAPI's native dependency tree and OpenAPI generation. +* **Web-Independent:** The `container.resolve()` method can be used in CLI tools, background workers, or unit tests without an active HTTP request. +* **Scoped Middleware:** Request-scoped dependencies (like `AsyncSession`) require the `ScopedDependencyMiddleware` to be active in `main.py`. This middleware initializes and clears the context-local storage for every request. --- -## πŸ’» Practical Usage +## Under-the-Hood Spec -### 1. Registering Custom Lifecycles -You can manually register bindings in your `plugin.py` or during application startup. +### 1. High-Performance Signature Caching +ZCore avoids the runtime cost of Python reflection. The first time a class is resolved, the container uses `inspect.signature` and `typing.get_type_hints` to analyze its dependencies [kernel/di.py]. This metadata is stored in `_dependency_signature_cache`, allowing subsequent resolutions to skip the expensive introspection phase. -```python -from zcore.kernel.di import container +### 2. Circular Dependency Safeguard +During resolution, the container maintains a recursion `_stack` [kernel/di.py]. If the container attempts to resolve a class that is already present in the current resolution stack, it raises a `CircularDependencyError` with a clear trace (e.g., `A -> B -> C -> A`), preventing a stack overflow. -# Register a globally shared Singleton -container.register_singleton(PaymentGateway, MockPaymentGateway()) +### 3. `Annotated` Metatype Resolution +The DI engine recognizes `typing.Annotated` during auto-wiring [kernel/di.py]. When a constructor parameter uses `Annotated[AsyncSession, Depends(get_db)]`, the engine strips the `Depends` wrapper by extracting the first argument of the `Annotated` origin using `get_args(annotation)[0]`. This allows ZCore to correctly resolve parameters typed with `Annotated` aliases like `SessionDep`, maintaining full compatibility with FastAPI's native dependency patterns. -# Register a Scoped implementation (Shared within one request) -container.register_scoped(IUserRepository, SQLUserRepository) -``` +### 4. Forward Reference Resolution +The DI engine utilizes `get_type_hints` rather than raw annotations. This allows ZCore to correctly resolve string-based forward references or `from __future__ import annotations` blocks, which standard `inspect` logic often fails to parse correctly. -### 2. Injecting Dependencies -We suggest using the `Inject` helper. It works seamlessly in both class constructors and FastAPI route handlers. +### 5. Context Isolation +Scoped instances are stored using `contextvars` [kernel/di.py]. This ensures that even in highly concurrent async environments, two different requests never share the same "Scoped" object, while still allowing all services within the *same* request to share the same instance (e.g., the same Database Transaction). +### 6. The `get_db` Dependency & `SessionDep` Alias +ZCore exposes a `get_db` async generator and a `SessionDep` type alias for seamless database session injection [db/setup.py]. `get_db` calls `container.resolve(AsyncSession)` through the IoC container, returning the request-scoped session managed by `ScopedDependencyMiddleware`. `SessionDep` is defined as `Annotated[AsyncSession, Depends(get_db)]`, providing a clean, reusable type annotation: ```python -from zcore.kernel.di import Inject -from products.repositories import ProductRepository +from zcore.db import SessionDep -class ProductService: - # ZCore will automatically provide the 'repository' instance - def __init__(self, repository: ProductRepository = Inject(ProductRepository)): - self.repository = repository +@router.get("/items") +async def list_items(db: SessionDep): + result = await db.execute(select(Item)) + return result.scalars().all() ``` +This eliminates per-endpoint `Depends(get_db)` boilerplate while keeping the injection point explicit and type-checkable. ---- - -## πŸ’‘ Engineering Insights - -!!! tip "πŸ’‘ Why use DI?" - DI makes your code **Testable**. In your real app, you inject a `PostgresRepository`. In your tests, you can inject a `MockRepository` without changing a single line of your `ProductService` logic. - -!!! info "πŸ›‘οΈ FastAPI Integration" - The `Inject` function is designed to be compatible with FastAPI's `Depends`. This means ZCore's DI and FastAPI's DI work together as one unified system. - -!!! warning "🧹 Scoped Cleanup" - Scoped dependencies are only as good as their cleanup. ZCore's `ScopedDependencyMiddleware` handles this automatically. If you are using the DI container outside of the web layer (e.g., in a background script), remember to manually manage your scope boundaries. +!!! info "Lifecycle Summary" + - **Singleton:** Created once per application lifetime. + - **Scoped:** Created once per HTTP request; shared across all services in that request. + - **Transient:** A fresh instance is created every time it is requested. diff --git a/docs/features/infrastructure/events.md b/docs/features/infrastructure/events.md index 00f5c18..71bc2da 100644 --- a/docs/features/infrastructure/events.md +++ b/docs/features/infrastructure/events.md @@ -1,119 +1,104 @@ -# πŸ”” Kernel Events System +# Asynchronous Event Orchestration -ZCore provides a modest and practical event system managed by the `EventDispatcher`. It enables "decoupled communication," which is a fancy way of saying it allows different parts of your application to talk to each other without being directly connected. This makes your code easier to maintain and extend. +Loosely coupled communication with concurrent, non-blocking execution and automated failure isolation. --- -## πŸ—οΈ The Event Dispatcher - -The central `EventDispatcher` acts as a "switchboard." It keeps track of who is interested in certain events and notifies them when those events occur. It is registered as a global singleton, so you can access it from anywhere in your application. - -### πŸ“‹ Core Methods - -| Method | Purpose | -| :--- | :--- | -| πŸ“₯ `subscribe` | Registers a function (listener) to wait for a specific event key. | -| πŸ“€ `dispatch` | Triggers an event and executes all registered listeners. | -| πŸ”Œ `unsubscribe` | Removes a listener so it no longer receives updates. | +
+
+ Type
+ Infrastructure Engine +
+
+ Status
+ Core Core +
+
+ Underlying Tech
+ asyncio / inspect +
+
+ +## The Challenge +In complex applications, triggering side effects often leads to **Tightly Coupled Spaghetti**. If a `UserService` needs to send a welcome email, notify a Slack channel, and initialize a loyalty account, it must import and call all those services directly. This results in: +1. **Testing Fragility:** To test a user registration, you must mock the email, slack, and loyalty services. +2. **Performance Bottlenecks:** Executing side effects sequentially increases the response time of the primary request. +3. **Failure Propagation:** If the email server is down and raises an exception, the entire user registration fails, even if the database write was successful. + +## The ZCore Elegance +The `EventDispatcher` enables a "Fire and Forget" architecture. Services simply dispatch an event, and the dispatcher handles the execution of all subscribers concurrently. Crucially, it isolates failuresβ€”an error in one listener will not stop other listeners or the main execution flow. + +=== "ZCore Event Dispatching" + :::python + from zcore import Inject, EventDispatcher + + class UserService: + def __init__(self, dispatcher: EventDispatcher = Inject(EventDispatcher)): + self.dispatcher = dispatcher + + async def register_user(self, schema): + # ... business logic ... + user = await self.repo.create(schema) + + # Dispatch and let the dispatcher handle the rest + await self.dispatcher.dispatch("user.registered", user=user) + return user + + # Subscribers are isolated in their own modules + @dispatcher.subscribe("user.registered") + async def send_welcome_email(user): + await email_service.send(user.email, "Welcome!") + +=== "FastAPI Manual Coupling" + :::python + # Logic is tightly bound and executed sequentially + class UserService: + async def register_user(self, schema): + user = await self.repo.create(schema) + + # 1. Sequential execution (Slow) + # 2. Tight coupling (Hard to test) + # 3. Direct failure risk (If email fails, registration fails) + try: + await self.email_service.send(user.email, "Welcome!") + await self.slack_service.notify(f"New user: {user.id}") + except Exception as e: + # Manual error handling required for every side effect + logger.error(e) + + return user --- -## πŸš€ Asynchronous Concurrent Execution - -When you trigger an event, ZCore doesn't just run the listeners one by one in a slow line. It intelligently analyzes each listener: - -1. **Synchronous Handlers:** Executed immediately in order. -2. **Asynchronous Handlers:** Grouped together and executed **concurrently** (at the same time) using `asyncio.gather`. This ensures that one slow task (like sending an email) doesn't block the rest of your application. +## Dispatcher Workflow +The `EventDispatcher` orchestrates a hybrid execution path for both synchronous and asynchronous listeners. -```mermaid -graph TD - Dispatch[dispatch_event] -->|Lookup| Reg[Subscriber Registry] - Reg -->|Identify Types| Split{Handler Type?} - Split -->|Async| AsyncList[Concurrent Pool] - Split -->|Sync| SyncList[Sequential Execution] - - subgraph Async Pool [Non-Blocking Pool] - AsyncList -->|asyncio.gather| Task1[Task 1] - AsyncList -->|asyncio.gather| Task2[Task 2] - end - - Task1 --> Result[Compiled Results] - Task2 --> Result - SyncList --> Result -``` +

+ Dispatcher Workflow +

--- -## πŸ›‘οΈ Fault Tolerance & Isolation - -A critical part of ZCore's engineering is **Fault Isolation**. In a modest system, if one "listener" fails (e.g., a notification service is down), it should not crash the entire process or prevent other listeners from working. - -ZCore uses a "Safe-Execute" pattern. Every listener is isolated so that errors are caught and logged without stopping the dispatcher: - -```python -# Internal safety mechanism -results = await asyncio.gather(*async_tasks, return_exceptions=True) -for res in results: - if isinstance(res, Exception): - # The error is logged, but execution continues - logger.error(f"Event listener failed: {res}") -``` +## Boundaries & Integration +The Event System is an optional orchestration bus that works alongside standard Python calls. -| Feature | Behavior | Benefit | -| :--- | :--- | :--- | -| **Error Trapping** | Caught via `return_exceptions=True`. | One failing task won't crash the others. | -| **Traceback Logging** | Full context recorded in logs. | Easy to debug without stopping production. | -| **Result Aggregation** | Returns results from all successful tasks. | Predictable feedback for the dispatcher. | +* **Singleton Registration:** The `Kernel` automatically registers the `EventDispatcher` as a singleton in the DI container. Any component can access it via `Inject(EventDispatcher)` [kernel/engine.py]. +* **Safe for Sync/Async:** You can subscribe with both standard `def` functions and `async def` coroutines. The dispatcher uses `inspect.iscoroutinefunction` to decide how to execute them [kernel/events.py]. +* **Bypass:** If you require a strict, sequential execution where the primary operation *must* fail if a side effect fails, do not use the dispatcher. Instead, call the service methods directly or use a `UnitOfWork` for transactional integrity. --- -## πŸ’» Practical Usage - -### 1. Subscribing to an Event -We suggest registering your event listeners inside your plugin's `on_startup` hook. This ensures they are ready as soon as the server starts. - -```python -from zcore.kernel import Plugin -from zcore.kernel.events import EventDispatcher -from zcore.kernel.di import Inject - -async def send_welcome_email(user_id: str) -> None: - # Logic to send email... - print(f"πŸ“§ Welcome email sent to {user_id}") - -class NotificationPlugin(Plugin): - name = "notifier" - - async def on_startup(self) -> None: - # Resolve dispatcher via DI and subscribe - dispatcher = Inject(EventDispatcher)() - dispatcher.subscribe("user.registered", send_welcome_email) -``` - -### 2. Dispatching an Event -You can trigger an event from any service or repository. - -```python -from zcore.kernel.events import EventDispatcher -from zcore.kernel.di import Inject - -async def register_user(user_data): - # ... logic to save user ... - - # Notify the system that a user was registered - dispatcher = Inject(EventDispatcher)() - await dispatcher.dispatch("user.registered", user_id="user_123") -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. Concurrent Execution Gather +When `dispatch()` is called, ZCore gathers all asynchronous subscribers and executes them concurrently using `asyncio.gather` [kernel/events.py]. This ensures that multiple side effects (like sending an email and hitting a webhook) do not increase the total response time of the request. -!!! tip "πŸ’‘ Non-Blocking by Design" - Since events are often used for "side-effects" (like logging or analytics), ZCore ensures they are non-blocking. Your main business logic continues to run while the event listeners work in the background. +### 2. Failure Isolation (`return_exceptions=True`) +The dispatcher is designed for stability. By using `return_exceptions=True` inside the gather call, ZCore ensures that if one handler raises an exception, the other handlers continue to run [kernel/events.py]. Exceptions are captured, logged via `structlog`, and the dispatcher returns `None` for the failed task instead of crashing the process. -!!! info "πŸ›‘οΈ Clean Unsubscription" - Always remember to `unsubscribe` if you are creating dynamic listeners that aren't tied to the global application lifecycle. For standard plugins, ZCore manages most of this for you. +### 3. Dynamic Handler Inspection +The dispatcher uses the `inspect` module to verify handlers during the dispatch phase [kernel/events.py]. It wraps the execution in a `try/except` block even during the "preparation" phase, catching errors that might occur if a handler is improperly configured or has an invalid signature. -!!! warning "🧹 Payload Consistency" - The dispatcher forwards all arguments (`*args`, `**kwargs`) to the listeners. Make sure your listener functions accept exactly the same parameters that the dispatcher sends to avoid `TypeError` errors. +!!! info "Transactional Events" + For database-related events, use the `UnitOfWork`. It buffers events and only calls `dispatcher.dispatch` **after** the database transaction has successfully committed [db/uow.py]. \ No newline at end of file diff --git a/docs/features/infrastructure/kernel.md b/docs/features/infrastructure/kernel.md index a8bdfb8..b839a01 100644 --- a/docs/features/infrastructure/kernel.md +++ b/docs/features/infrastructure/kernel.md @@ -1,67 +1,97 @@ -# 🧠 Framework Engine & Plugin Protocol +# Framework Engine & Orchestration -The ZCore `Kernel` is the modest orchestration engine at the heart of every application. It views your software not as a single block of code, but as a collection of self-contained **Plugins**. The Kernel's primary job is to coordinate these plugins, ensuring they are configured correctly and initialized in the exact order their dependencies require. +The central nervous system coordinating plugin lifecycles, dependency resolution, and modular application transitions. --- -## πŸ“œ The Plugin Contract (Protocol) - -In ZCore, a "Plugin" is any module that follows a specific set of rules (a Protocol). This structure ensures that whether you are building a simple logger or a complex payment system, the framework knows exactly how to talk to your code. - -| Method | Timing | Practical Use Case | -| :--- | :--- | :--- | -| πŸ› οΈ `setup(app)` | **Immediate** | Registering routes and binding DI singletons. | -| 🏁 `before_startup` | **Pre-flight** | Verifying environment variables or file permissions. | -| πŸ”₯ `on_startup` | **Bootstrapping** | Opening database pools or warming up caches. | -| ✨ `after_startup` | **Post-launch** | Triggering health checks or background workers. | -| πŸ›‘ `on_shutdown` | **Cleanup** | Safely closing connections and flushing buffers. | +
+
+ Type
+ Core Orchestrator +
+
+ Status
+ Core Infrastructure +
+
+ Underlying Tech
+ graphlib / Protocol +
+
+ +## The Challenge +As FastAPI projects grow, the initialization sequence becomes a "procedural soup" inside `main.py`. Developers often encounter: +1. **Race Conditions:** A service attempting to warm up a cache before the database connection pool is initialized. +2. **Circular Import Deadlocks:** Modules importing each other's setup functions during startup. +3. **Missing Requirements:** An application starting partially even when critical infrastructure (like a message broker plugin) failed to register. +4. **Tear-down Leaks:** External connections (Redis, DB) not being closed in the correct order during shutdown, causing hanging processes. + +## The ZCore Elegance +The `Kernel` manages the application through a **Topological Plugin System**. Every architectural unit is treated as a `Plugin` with declared dependencies. The Kernel calculates the mathematically correct initialization order using a Directed Acyclic Graph (DAG), ensuring every component is ready before the next one starts. + +=== "ZCore Topological Plugins" + :::python + from zcore import Plugin, Kernel + + class PaymentPlugin(Plugin): + name = "payments" + # Explicitly wait for DB and Logging + dependencies = ["db", "logging"] + + def setup(self, app: FastAPI): + app.include_router(payment_router) + + async def on_startup(self): + await gateway.connect() + + # Kernel sorts and executes in the right order + kernel = Kernel() + kernel.add_plugin(PaymentPlugin()) + kernel.add_plugin(DatabasePlugin()) # name="db" + +=== "FastAPI Procedural Lifespan" + :::python + # Manual ordering is fragile and hard to track + @asynccontextmanager + async def lifespan(app: FastAPI): + # You must manually ensure 'db' is before 'payments' + await init_db() + await init_logging() + + # If this is moved up by mistake, the app crashes + await gateway.connect() + yield + # Manual shutdown is also tedious + await gateway.close() + await db.close() --- -## πŸ“ Topological Dependency Resolution - -One of the most practical features of the Kernel is how it handles dependencies. If your `AnalyticsPlugin` needs the `DatabasePlugin` to be ready first, you simply declare it. - -The Kernel uses a mathematical approach called **Topological Sorting** to build a Directed Acyclic Graph (DAG). This ensures that no matter how many plugins you have, they always start in a sequence where every "parent" is ready before the "child" begins. +## Boundaries & Integration +The Kernel is a thin orchestration layer that respects the native FastAPI lifecycle. -```mermaid -graph TD - DB[1. Database Connection] -->|Required By| Auth[2. Identity & Auth] - Auth -->|Required By| Products[3. Product Catalog] - Auth -->|Required By| Orders[4. Order Management] - Products -->|Required By| Search[5. Search Indexer] -``` - -### πŸ›‘οΈ Built-in Engineering Safeguards: -1. **Cyclic Detection:** If Plugin A depends on B, and B depends on A, the Kernel will detect this "infinite loop" and abort startup with a clear error message. -2. **Missing Requirements:** If a plugin asks for a dependency that isn't registered, the Kernel stops immediately, preventing "partial" system failures that are hard to debug. +* **Native Lifespan:** The `kernel.lifespan` method is a standard Python `@asynccontextmanager`. It is passed directly to the `FastAPI` constructor, meaning the framework follows the standard Starlette execution path. +* **Protocol-Based:** You don't need to inherit from a specific ZCore class; as long as your object satisfies the `Plugin` protocol (having `name`, `version`, `dependencies`, and lifecycle methods), the Kernel will accept it [kernel/plugins.py]. +* **Bypass:** If a small part of your app doesn't need modularity, you can still register routes or logic directly in `main.py` using standard FastAPI methodsβ€”ZCore will ignore these and only manage the registered plugins. --- -## πŸ”„ The Lifespan Sequence - -The Kernel manages the application lifecycle through a FastAPI context manager. It ensures that startup happens in **forward order** (following dependencies), but shutdown happens in **reverse order** to release resources safely. +## Under-the-Hood Spec -```mermaid -flowchart TD - Start([Server Boot]) --> Setup[1. Synchronous setup] - Setup --> B[2. before_startup] - B --> S[3. on_startup] - S --> A[4. after_startup] - A --> Running{App Active} - Running --> Stop[5. on_shutdown - REVERSE ORDER] - Stop --> End([Server Stop]) -``` - ---- +### 1. DAG-Based Dependency Resolution +The Kernel utilizes Python's `graphlib.TopologicalSorter` to analyze the `dependencies` list of every plugin [kernel/engine.py]. If Plugin A requires Plugin B, the sorter ensures B is processed first. If a circular dependency is detected (e.g., A -> B and B -> A), ZCore raises a `CycleError` during the `setup` phase, preventing the application from starting in an unstable state. -## πŸ’‘ Engineering Insights +### 2. Reverse-Topological Shutdown +Resource management requires a "First In, Last Out" approach. The Kernel ensures that the `on_shutdown` hooks are executed in the **reverse order** of the startup sequence [kernel/engine.py]. This ensures that shared resources (like the Database) are only closed after the domain modules that depend on them have finished their cleanup tasks. -!!! tip "πŸ’‘ Safe Resource Teardown" - By running `on_shutdown` in reverse order, ZCore ensures that the `DatabasePlugin` is the **last** thing to close. This allows other plugins to finish their final database writes before the connection pool is destroyed. +### 3. Missing Dependency Assertion +During the resolution phase, the Kernel cross-references every declared dependency name against the active registry [kernel/engine.py]. If a plugin declares a dependency that hasn't been registered in the `Kernel`, ZCore halts initialization with a `RuntimeError`, identifying the exact missing dependency by name. -!!! info "πŸ›‘οΈ Error Resilience" - ZCore follows a **Fail-Fast** philosophy. If any plugin fails during its `on_startup` phase, the Kernel aborts the entire sequence and shuts down the server. This prevents your application from running in a "half-broken" state where some features work and others don't. +### 4. Triple-Wave Startup +The Kernel executes startup logic in three distinct, sequential waves across all sorted plugins to ensure total system stability [kernel/engine.py]: +1. **`before_startup`**: Verification and pre-checks. +2. **`on_startup`**: Primary resource initialization. +3. **`after_startup`**: Post-initialization signaling or background task spawns. -!!! note "🧠 Why a Protocol?" - ZCore uses `typing.Protocol` instead of standard inheritance for plugins. This means your class doesn't *have* to inherit from anything to be a plugin; it just needs to have the right methods. This keeps your code clean and reduces "framework coupling." +!!! info "Kernel Events" + The Kernel automatically initializes a central `EventDispatcher` and registers it as a singleton in the DI container during the `setup` phase. This allows plugins to communicate through loosely coupled events without direct imports. \ No newline at end of file diff --git a/docs/features/infrastructure/main.md b/docs/features/infrastructure/main.md index 7fee14d..b13af64 100644 --- a/docs/features/infrastructure/main.md +++ b/docs/features/infrastructure/main.md @@ -1,125 +1,98 @@ -# πŸš€ Application Bootstrapping +# Application Entry Point -Bootstrapping is the modest but crucial process of assembling your application's components into a single, running unit. In ZCore, this happens in `main.py`. It is the "meeting place" where logging, database connections, security layers, and your domain plugins are wired together. +The centralized orchestrator where infrastructure, security context, and domain plugins converge into a unified FastAPI application. --- -## πŸ“ The Assembly Sequence - -A predictable startup sequence is essential for system stability. ZCore follows a strict order of operations to ensure that each layer is ready before the next one relies on it. - -```mermaid -graph TD - 1[1. setup_logging] --> 2[2. db_manager.init_app] - 2 --> 3[3. Initialize Kernel] - 3 --> 4[4. Add Domain Plugins] - 4 --> 5[5. Register DB Event Dispatcher] - 5 --> 6[6. Initialize FastAPI with Kernel Lifespan] - 6 --> 7[7. Register Middlewares] - 7 --> 8[8. Register Exception Handlers] - 8 --> 9[9. Setup Plugins kernel.setup] -``` +
+
+ Type
+ Bootstrap Orchestrator +
+
+ Status
+ Core Requirement +
+
+ Underlying Tech
+ FastAPI / ASGI / ZCore Kernel +
+
+ +## The Challenge +In standard FastAPI projects, the `main.py` file frequently evolves into a "God File"β€”a massive procedural script handling everything from logging and database initialization to middleware ordering and raw router includes. This results in: +- **Procedural Fragility:** Changing the order of a single line (e.g., placing Auth middleware after Routing) causes silent, critical failures. +- **Initialization Race Conditions:** Background tasks or services attempting to run before the database connection pool is warmed up. +- **Tight Coupling:** No clean way to enable or disable features (like a Search plugin or Caching) without manually hacking the core entry file. + +## The ZCore Elegance +ZCore provides a declarative bootstrap structure via the `MAIN_PY_TEMPLATE` [cli/templates.py]. Instead of writing procedural logic, you configure the `Kernel`, register your domain `Plugins`, and attach core architectural Middlewares. The `main.py` serves strictly as an **Orchestrator**, ensuring infrastructure is ready before any domain logic begins. + +=== "ZCore Declarative main.py" + :::python + from fastapi import FastAPI + from zcore import Kernel, settings + from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware + from zcore.db import db_manager, register_db_event_dispatcher + from zcore.logging import setup_logging + + # 1. Warm up Infrastructure + setup_logging() + db_manager.init_app(db_url=settings.DATABASE_URL) + + # 2. Initialize Kernel & Event Bridge + kernel = Kernel() + register_db_event_dispatcher(kernel.dispatcher) + + # 3. Create App with Kernel Lifespan + app = FastAPI(title=settings.PROJECT_NAME, lifespan=kernel.lifespan) + + # 4. Bind Plugins & Middleware + kernel.setup(app) + app.add_middleware(RequestLogMiddleware) + app.add_middleware(ScopedDependencyMiddleware) + +=== "Standard FastAPI Procedural" + :::python + # Standard projects mix infra and business logic procedurally + app = FastAPI() + + @app.on_event("startup") + async def startup(): + # Manual, fragile ordering + await init_logging() + await init_db() + # What if this fails? Is the error caught? + await some_service_warmup() + + # Router includes are often scattered + from orders.routers import router as orders_router + app.include_router(orders_router) + + # Middleware ordering is manual and invisible + app.add_middleware(MyMiddleware) --- -## πŸ› οΈ Bootstrapping Reference Table +## Boundaries & Integration +The entry point is designed to wrap FastAPI's power without obscuring its standard API. -| Step | Component | Practical Purpose | -| :--- | :--- | :--- | -| **1** | **Logging** | Standardizes diagnostics so logs are readable by both humans and machines. | -| **2** | **Database** | Prepares the async connection pool and sets performance boundaries. | -| **3 & 4** | **Kernel** | Registers your domain modules and calculates their startup order. | -| **5** | **DB Bridge** | Allows database transactions to communicate with the rest of the system. | -| **6** | **FastAPI** | Creates the web engine and binds it to the Kernel’s lifecycle hooks. | -| **7** | **Middlewares** | Sets up request tracing and ensures isolated dependency sandboxes. | -| **8** | **Error Handling** | Standardizes how failures are communicated back to the client. | -| **9** | **Setup** | Finalizes the wiring (e.g., registering routes for every plugin). | +- **Native App Instance:** The `app` variable is a standard `FastAPI()` instance. You can use any native FastAPI decorator (`@app.get`) or feature directly in `main.py` without restriction. +- **Middleware Freedom:** ZCore provides `RequestLogMiddleware` and `ScopedDependencyMiddleware` for architectural stability, but you can add any third-party Starlette/FastAPI middleware using the standard `app.add_middleware()` method. +- **Plugin Decoupling:** You can add or remove domain modules simply by calling `kernel.add_plugin(MyModulePlugin())`. The Kernel handles the routing registration and startup logic internally. --- -## πŸ’» Complete Assembly Guide (`main.py`) - -We suggest the following structured layout for your entry point to ensure all ZCore features are fully utilized: - -```python -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from fastapi import FastAPI -from zcore import Kernel, settings -from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware -from zcore.exceptions import app_exception_handler, AppException -from zcore.db import db_manager, register_db_event_dispatcher -from zcore.logging import setup_logging - -# Import your custom modular plugins -from products.plugin import ProductPlugin -from billing.plugin import BillingPlugin - -# 1. Initialize Structured Logging -# Standardizes logs into a consistent format (Console or JSON). -setup_logging() - -# 2. Configure the Database Manager -# Tunes the async connection pool according to your .env settings. -db_manager.init_app( - db_url=settings.DATABASE_URL, - pool_size=settings.POOL_SIZE, - max_overflow=settings.MAX_OVERFLOW, - echo=(settings.ENVIRONMENT == "development") -) - -# 3. Create the Core Kernel -kernel = Kernel() - -# 4. Register Modular Domain Plugins -# The Kernel will automatically sort these based on their dependencies. -kernel.add_plugin(ProductPlugin()) -kernel.add_plugin(BillingPlugin()) - -# 5. Register the Global Database Event Dispatcher -# Enables "Post-Commit" events across your repositories and services. -register_db_event_dispatcher(kernel.dispatcher) - -# 6. Initialize FastAPI and Hook the Kernel Lifespan -# This connects ZCore's lifecycle hooks (on_startup, etc.) to the web server. -app = FastAPI( - title=settings.PROJECT_NAME, - lifespan=kernel.lifespan -) - -# 7. Apply Core Architectural Middlewares -# Handles Request Correlation IDs and Scoped DI cleanup. -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) - -# 8. Apply the Centralized Exception Handler -# Ensures business failures are returned as clean, standardized JSON. -app.add_exception_handler(AppException, app_exception_handler) - -# 9. Finalize Plugin Setup -# Triggers each plugin's setup() hook to register its routes and routers. -kernel.setup(app) - -@app.get("/health") -async def health(): - return { - "status": "healthy", - "framework": "ZCore", - "environment": settings.ENVIRONMENT - } -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The DB-Event Dispatcher Bridge +A unique architectural feature is `register_db_event_dispatcher(kernel.dispatcher)` [db/events.py]. This creates a secure link between low-level database lifecycle events (like transaction commits) and the high-level application `EventDispatcher`. It allows domain handlers to respond to database changes in a non-blocking, loosely coupled manner. -!!! tip "πŸ’‘ Debugging SQL Queries" - By setting `echo=(settings.ENVIRONMENT == "development")`, SQLAlchemy will print every compiled SQL statement to your terminal. This is a practical way to detect "N+1 query" problems early in development. +### 2. Path Injection for Modular Imports +ZCore's `main.py` template includes a `sys.path.insert(0, ...)` block [cli/templates.py]. This ensures that your modular apps (generated via `zc startapp`) are discoverable as top-level modules, preventing complex relative import issues common in multi-directory FastAPI structures. -!!! info "πŸ›‘οΈ Middleware Order" - The order of `app.add_middleware` is important. The `RequestLogMiddleware` is registered first so it can generate a unique `request_id` that all other middlewares and services can use for logging. +### 3. Integrated Exception Handling +The entry point registers the `app_exception_handler` [exceptions/handlers.py]. This ensures that any `AppException` (EntityNotFound, ValidationError, etc.) raised deep within a Service or Repository is caught at the boundary, logged with structured metadata, and returned in the standardized `ResponseWrapper` JSON envelope. -!!! note "🧠 Minimalist Main" - A well-engineered `main.py` should be mostly declarative. Its job is to "register" and "configure," not to contain actual business logic. If this file grows too large, consider moving configuration logic into separate plugins. +!!! info "Middleware Ordering" + In ZCore, `RequestLogMiddleware` should generally be added first to ensure the entire request duration (including DI resolution time) is captured and a correlation ID is generated before any other logic executes. \ No newline at end of file diff --git a/docs/features/infrastructure/middleware.md b/docs/features/infrastructure/middleware.md index 5c41766..37a859b 100644 --- a/docs/features/infrastructure/middleware.md +++ b/docs/features/infrastructure/middleware.md @@ -1,77 +1,80 @@ -# πŸ›‘οΈ ASGI Web Middlewares +# ASGI Lifecycle Coordination -ZCore provides two modest but essential ASGI middlewares that coordinate the lifecycle of every HTTP request. Their primary job is to ensure that requests are traceable, logs are structured, and dependencies remain safely isolated in their own "sandboxes." +Trace every transaction with correlation IDs and manage request-scoped state boundaries for safe dependency resolution. --- -## 1. Request Tracking & Traceability (`RequestLogMiddleware`) - -The `RequestLogMiddleware` is the first point of contact for an incoming request. It manages "Correlation IDs" and binds metrics to your logs so you can trace exactly what happened during a specific transaction. - -### πŸ“ Execution Flow -This diagram illustrates how the middleware handles the identity and timing of a request: - -```mermaid -graph TD - Request[Incoming HTTP Request] -->|Inspect Headers| Match{x-request-id Found?} - Match -->|Yes| Pattern{Valid Pattern?} - Pattern -->|Matches Regex| Apply[Bind to Context] - Pattern -->|Invalid| Gen[Generate Fresh UUIDv4] - Match -->|No| Gen - Gen --> Apply - Apply -->|Execute Downstream| App[Business Logic] - App -->|Measure Duration| Log[Log Request Metrics] - Log -->|Append Header| Response[Outgoing Response] -``` - -### πŸ›‘οΈ Modest Security Precautions -To prevent "Log Injection" attacks where a malicious client sends a carefully crafted header to corrupt your logs, ZCore enforces a strict character pattern. Only alphanumeric characters and a few safe symbols are allowed: - -```python -# Modest regex safety check -REQUEST_ID_PATTERN = re.compile(r"^[a-zA-Z0-9\-\.\_\:]{8,64}$") -``` - -### πŸͺ΅ Structured Logging Context -Once a valid ID is resolved, it is bound to the `structlog` context variables. This means every log entry generated during that requestβ€”whether it's from a service, a repository, or a pluginβ€”will automatically include the `request_id`, making debugging significantly easier. +
+
+ Type
+ ASGI Middleware +
+
+ Status
+ Recommended Core +
+
+ Underlying Tech
+ Starlette / contextvars / structlog +
+
+ +## The Challenge +Maintaining observability and data integrity in an asynchronous environment presents unique challenges: +1. **Disconnected Logs:** Without a Correlation ID (Request ID), tracking a single user's request journey through multiple service layers in a log aggregator is nearly impossible. +2. **Session Leakage:** Improperly managed database sessions can lead to connection leaks or, worse, different services within the same request accidentally using different transactions. +3. **Manual Context Passing:** Developers often find themselves passing a "context" dictionary or a raw database session object through every function call just to ensure a shared state. + +## The ZCore Elegance +ZCore provides a duo of high-performance ASGI middlewares that automate these concerns at the entry point. `RequestLogMiddleware` ensures every request is uniquely identifiable and timed, while `ScopedDependencyMiddleware` initializes a secure execution boundary where the database session and other scoped dependencies are automatically shared and cleaned up. + +=== "ZCore Middleware Registration" + :::python + from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware + + # Standard order in main.py + app.add_middleware(RequestLogMiddleware) + app.add_middleware(ScopedDependencyMiddleware) + +=== "FastAPI Manual Management" + :::python + # Replicating ZCore's tracing and scoping manually: + @app.middleware("http") + async def trace_and_scope(request: Request, call_next): + request_id = request.headers.get("x-request-id", str(uuid.uuid4())) + # Manually bind to logger + # Manually start a timer + # Manually open a DB session + async with session_factory() as session: + # How do we pass this session to nested services + # without passing it as an argument everywhere? + response = await call_next(request) + return response --- -## 2. Dependency Isolation (`ScopedDependencyMiddleware`) - -The `ScopedDependencyMiddleware` is responsible for creating and destroying the "Request Sandbox." It ensures that dependencies like database sessions are isolated to a single request and are safely released when the request ends. - -### πŸ“¦ The Request Sandbox Lifecycle -This middleware coordinates with the **IoC Container** to manage the `Scoped` lifecycle. +## Boundaries & Integration +ZCore Middlewares follow the standard Starlette/ASGI protocol, ensuring they play well with the rest of the ecosystem. -| Phase | Action | Why it Matters | -| :--- | :--- | :--- | -| 🏁 **Request Entry** | Generate a unique `scope_id`. | Identifies the unique "sandbox" for this specific request. | -| πŸ’‰ **Initialization** | Bind `scope_id` to `contextvars`. | Tells the DI container where to store newly created objects. | -| 🚦 **Execution** | Run the application logic. | Any `Scoped` dependency (like a DB session) is shared only within this request. | -| 🧹 **Teardown** | `container.clear_scope()` | **Critical:** Safely closes and purges all objects created during the request. | +* **Standard ASGI:** Both components are pure ASGI implementations. You can use them in any FastAPI, Starlette, or even raw ASGI application. +* **Structlog Integration:** `RequestLogMiddleware` is designed to work with ZCore's structured logging. It automatically binds the `request_id` to `structlog.contextvars`, making it visible in every log line generated during that request. +* **Bypass:** If you prefer a different logging or session management strategy, you can simply omit these middlewares. ZCore's internal components (like `Inject`) will still function, but "Scoped" lifecycles will fallback to "Transient" behavior if no active scope is detected. -### πŸ› οΈ Practical Implementation -ZCore handles this automatically in the background. Here is a modest look at the internal cleanup logic: +--- -```python -try: - # Process the request - await self.app(scope, receive, send) -finally: - # Guarantee cleanup even if an error occurred - container.clear_scope(scope_id) -``` +## Under-the-Hood Spec ---- +### 1. Hardened Correlation IDs +The `RequestLogMiddleware` uses a strict regex pattern (`REQUEST_ID_PATTERN`) to validate incoming `x-request-id` headers [web/middleware.py]. If a header is present but malformed or too long, ZCore rejects it and generates a fresh `uuid4` instead. This prevents header-injection attacks or bloated log metadata. -## πŸ’‘ Engineering Insights +### 2. The `send_wrapper` Interceptor +ZCore doesn't just log the request ID; it ensures the client receives it back. The middleware wraps the ASGI `send` callable [web/middleware.py]. When the application starts sending the response (`http.response.start`), ZCore intercepts the event and appends the `x-request-id` to the outgoing headers, facilitating easier frontend-to-backend debugging. -!!! tip "πŸ’‘ Observability Tip" - If your system interacts with other microservices, ensure they also forward the `x-request-id` header. This allows ZCore to link its logs with the logs of other systems, providing a complete "Distributed Trace" of a user's journey. +### 3. Atomic Scope Cleanup +`ScopedDependencyMiddleware` generates a unique `scope_id` for every request and sets it in a `contextvar` [web/middleware.py]. Most importantly, it creates a database session using `db_manager.session()` and registers it as a scoped instance in the DI container. The `finally` block guarantees that `container.clear_scope(scope_id)` is called even if the request crashes, ensuring zero memory leaks or hanging DB connections. -!!! warning "πŸ›‘οΈ Middleware Order" - In your `main.py`, always register `RequestLogMiddleware` **before** other middlewares. This ensures that the `request_id` is available for all subsequent layers, including the `ScopedDependencyMiddleware`. +### 4. Precision Timing +Logging uses `time.perf_counter()` to calculate the `duration_ms` [web/middleware.py]. This provides sub-millisecond precision for performance monitoring, which is then logged alongside the HTTP method and path as a structured record. -!!! info "πŸ›‘οΈ Preventing Memory Leaks" - The `ScopedDependencyMiddleware` is your primary defense against memory leaks. By calling `clear_scope` in a `finally` block, ZCore ensures that database connections are returned to the pool and memory is released even if the application crashes during a request. +!!! info "Middleware Ordering" + It is recommended to place `RequestLogMiddleware` as the outermost layer to capture the total execution time, including the time spent in subsequent middlewares like `ScopedDependencyMiddleware`. \ No newline at end of file diff --git a/docs/features/infrastructure/utils.md b/docs/features/infrastructure/utils.md index 61c42f2..a45e36f 100644 --- a/docs/features/infrastructure/utils.md +++ b/docs/features/infrastructure/utils.md @@ -1,84 +1,91 @@ -# πŸ› οΈ System Utilities +# System Utilities -ZCore provides a modest and practical set of system utilities to handle the "small but important" tasks that every application needs. These tools ensure that operations like JSON serialization, data validation, and text transformation are handled consistently across all your modules. +Low-level data transformation, secure JSON serialization, and dynamic schema validation utilities. --- -## 1. Advanced JSON Serialization (`CustomJSONEncoder`) - -Standard Python JSON libraries often struggle with common data types like `Decimal`, `UUID`, and `datetime`. ZCore solves this by extending the default encoder to support these types out of the box, preventing unexpected `TypeError` errors when returning data. - -| Data Type | Target Format | Implementation Strategy | -| :--- | :--- | :--- | -| πŸ†” **`uuid.UUID`** | `str` (String) | Converted via `str(obj)`. | -| πŸ“… **`datetime / date / time`** | `str` (ISO-8601) | Converted via `obj.isoformat()`. | -| πŸ’° **`Decimal`** | `float` (Float) | Cast via `float(obj)` for JSON compatibility. | - -### πŸ’» Practical Usage -We suggest using ZCore's `json_dumps` and `json_loads` wrappers instead of the standard library to ensure these rules are always applied. - -```python -import uuid -from decimal import Decimal -from zcore.utils.helpers import json_dumps - -payload = { - "id": uuid.uuid4(), - "amount": Decimal("150.75"), - "status": "active" -} - -# Safely serialize to a JSON string without errors -json_str = json_dumps(payload) -``` +
+
+ Type
+ Utility Layer +
+
+ Status
+ Core Infrastructure +
+
+ Underlying Tech
+ json / jsonschema / re +
+
+ +## The Challenge +Modern web applications frequently handle non-primitive data types (UUIDs, Decimals, Datetime objects) that the standard Python `json` library cannot serialize by default. Developers often resort to manual string casting or scattered `isoformat()` calls. + +Additionally, validating dynamic JSON payloads against schemas often requires repetitive `try-except` blocks to catch `jsonschema` errors and translate them into user-friendly API responses, leading to "Utility Spaghetti" across the codebase. + +## The ZCore Elegance +ZCore provides a robust set of helpers to standardize data handling. The `CustomJSONEncoder` automatically handles all architectural types, while `json_dumps` and `json_loads` provide a safe, high-level interface. The `validate_json_schema` utility integrates directly with ZCore’s exception system, translating structural violations into standardized `ValidationError` responses. + +=== "ZCore Utility Handling" + :::python + from zcore.utils.helpers import json_dumps, slugify + from zcore.utils.json_schema import validate_json_schema + + # 1. Automatic serialization of complex types + data = {"id": uuid.uuid4(), "price": Decimal("19.99"), "at": datetime.utcnow()} + raw_json = json_dumps(data) + + # 2. Dynamic Schema Validation with auto-error mapping + schema = {"type": "object", "required": ["name"]} + validate_json_schema({"age": 25}, schema) # Raises AppException(400) + + # 3. Standardized URL slugs + url_slug = slugify("ZCore: Modern Framework!") # "zcore-modern-framework" + +=== "Raw Python Implementation" + :::python + # Manual conversion for standard json library + import json + + data = {"id": uuid.uuid4(), "at": datetime.utcnow()} + + # Standard json.dumps(data) fails with TypeError + raw_json = json.dumps(data, default=lambda o: str(o) if isinstance(o, uuid.UUID) else o.isoformat()) + + # Manual jsonschema validation + from jsonschema import validate, ValidationError as JSValidationError + try: + validate(instance=payload, schema=schema) + except JSValidationError as e: + # Manual translation to FastAPI responses + raise HTTPException(status_code=400, detail=str(e)) --- -## 2. Dynamic JSON Schema Validation (`validate_json_schema`) - -This utility allows you to validate raw JSON structures against **Draft-7** standards. It is particularly useful when handling dynamic data that isn't captured by your fixed Pydantic models. +## Boundaries & Integration +ZCore Utilities are built as thin, non-intrusive wrappers over standard libraries. -```mermaid -graph TD - Input[JSON Document Input] -->|Check Parameters| SchemaNull{Schema Defined?} - SchemaNull -->|No| MetaCheck[Validate Data as a valid Schema itself] - SchemaNull -->|Yes| Validate[Validate Data against provided Schema] - MetaCheck -->|Invalid| Error[🚨 Raise ValidationError] - Validate -->|Fails| Error - Validate -->|Success| Pass[βœ… Continue] -``` - -### πŸ“‹ Two Modes of Operation: -1. **Data Validation:** Checks if a piece of data (like a configuration file or a dynamic form submission) matches your rules. -2. **Schema Verification:** If you are building a tool that *generates* schemas, call this function with only the schema. It will verify that your schema is structurally sound and follows Draft-7 rules. +* **Pydantic Integration:** While Pydantic handles its own serialization, ZCore's utilities are essential for low-level caching (Redis), logging, and WebSockets where raw Pydantic objects are not present [utils/helpers.py]. +* **Draft-7 Standards:** The `validate_json_schema` utility follows the JSON Schema Draft-7 meta-schema, ensuring industry-standard compatibility [utils/json_schema.py]. +* **Bypass:** You can continue to use Python's native `json` or `re` modules at any time. ZCore does not override global built-ins; it provides explicit alternatives for framework-specific needs. --- -## 3. URL-Safe Slug Generation (`slugify`) - -The `slugify` utility is a modest tool that transforms human-readable text into "URL-safe" strings. This is ideal for generating clean URLs from titles or names. - -### πŸ“ Transformation Logic: -* Converts all text to **lowercase**. -* Removes special characters and symbols. -* Replaces spaces and underscores with a **single hyphen**. -* Trims leading and trailing hyphens. - -| Input String | Resulting Slug | -| :--- | :--- | -| `My Awesome Product!` | `my-awesome-product` | -| `Laptop _ SKU-123` | `laptop-sku-123` | -| `---Clean Me---` | `clean-me` | - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The Custom Serialization Pipeline +The `CustomJSONEncoder` implements a fallback mechanism [utils/helpers.py]. It prioritizes high-performance checks for `uuid.UUID`, `Decimal`, and `datetime/date/time` objects. If a type is still unrecognized, it performs a final `str()` cast to ensure the `json_dumps` operation never raises a `TypeError`, which is critical for logging reliability. -!!! tip "πŸ’‘ Why use Decimals in JSON?" - While JSON doesn't have a native "Decimal" type, ZCore converts them to floats during serialization. This is a practical compromise for web communication. However, always remember that for financial calculations *inside* your Python code, you should keep them as `Decimal` objects to preserve precision. +### 2. Schema Meta-Validation +When `validate_json_schema` is called with only one argument, it performs **Meta-Validation** [utils/json_schema.py]. It uses `Draft7Validator.check_schema(data)` to verify that the provided dictionary is itself a structurally valid JSON Schema. This is used internally by the framework to validate dynamically generated schemas before they are served or processed. -!!! info "πŸ›‘οΈ Error Context" - When `validate_json_schema` fails, it doesn't just say "No." It returns a structured `ValidationError` containing the exact `path` to the problematic field and the specific `schema` rule that was violated, making it much easier for your API clients to fix their request. +### 3. Sanitized Slugification +The `slugify` utility uses a multi-pass regex strategy [utils/helpers.py]: +1. Removes non-alphanumeric characters. +2. Normalizes spaces and underscores into single dashes. +3. Trims leading/trailing dashes. +This ensures that generated slugs are always URL-safe and conform to SEO best practices without manual character mapping. -!!! warning "🧹 Slug Sanitization" - The `slugify` tool uses regular expressions to strip unsafe characters. It is designed for URL paths and filenames. While it is robust, always ensure you handle potential collisions (e.g., two products named "Widget") by appending a unique ID or hash if necessary. +!!! info "Performance Note" + ZCore's `json_loads` and `json_dumps` pass through additional `**kwargs` to the underlying `json` module, allowing you to use optimizations like `separators` or `sort_keys` while still benefiting from ZCore's enhanced type support. \ No newline at end of file diff --git a/docs/features/security/api-router.md b/docs/features/security/api-router.md index 6dc166d..a5d58b6 100644 --- a/docs/features/security/api-router.md +++ b/docs/features/security/api-router.md @@ -1,79 +1,101 @@ -# 🚦 API Routing & Dynamic Schema Pruning +# API Routing Mechanics -ZCore extends FastAPI's routing capabilities through a modest but powerful subclass called `ZCoreAPIRoute`. This specialized route handler acts as an intelligent "gatekeeper," managing how data structures are exposed to the outside world and ensuring that security policies are enforced even at the metadata level. +Intercept HTTP traffic with a custom route handler that preserves response classes, injects Vary headers on context-pruned payloads, and exposes dynamic JSON-Schema introspection. --- -## πŸ” Dynamic Schema Pruning (`?schema=true`) +
+
+ Type
+ Routing Infrastructure +
+
+ Status
+ Security Integration +
+
+ Underlying Tech
+ FastAPI APIRoute / Pydantic +
+
-A practical challenge in modern web development is keeping the frontend in sync with backend validation rules. ZCore allows clients to query an endpoint's structure by appending `?schema=true` to any supported URL. +## The Challenge -However, exposing raw schemas can be a security risk if they reveal restricted fields. ZCore solves this by leveraging the `Zchema` base class, which automatically prunes the JSON Schema during generation β€” before it ever leaves the server. +Standard FastAPI routes are stateless with respect to security context. As a result, developers often encounter: -### πŸ“ The Pruning Pipeline +1. **Response Class Overwriting:** Middleware or custom response classes are silently replaced by `JSONResponse`, breaking specialized serialization logic. +2. **Cache Invisibility:** When the response payload is dynamically pruned based on user permissions, intermediate CDNs or proxies may cache a pruned response and serve it to a privileged user, leaking restricted data. +3. **Schema Blindness:** Frontend teams have no standardized way to introspect the current schemaβ€”especially which fields are restricted for the active userβ€”without manual API documentation or hard-coded logic. -```mermaid -graph TD - Request[Client Request: GET /products/?schema=true] -->|Verify Permissions| IsExposed{Exposed?} - IsExposed -->|No| Reject[🚨 400 Bad Request] - IsExposed -->|Yes| Generate[Generate JSON Schema via Zchema.__get_pydantic_json_schema__] - Generate -->|Resolve __db_name__ context| Prune[Prune properties matching restricted fields] - Prune -->|Remove properties & requirements| Output[Return Pruned Schema JSON] -``` +## The ZCore Elegance -### 🧠 How It Works +`ZCoreAPIRoute` is a drop-in replacement for FastAPI's `APIRoute`. It intercepts every request, conditionally generates context-aware JSON-Schema responses when `?schema=true` is appended, and appends `Vary: Authorization, Cookie` headers to any response that has been security-pruned. -When `?schema=true` is requested, `ZCoreAPIRoute` calls `model_json_schema()` on the target schema. Because the schema inherits from `Zchema`, the `__get_pydantic_json_schema__` hook is triggered automatically. This hook: +=== "ZCore Transparent Routing" + :::python + from zcore.web.api_router import ZCoreAPIRoute + from fastapi import APIRouter -1. Resolves the active context's restricted fields via `get_restricted_fields()` -2. Filters the paths to only those matching the schema's `__db_name__` -3. Recursively prunes matching properties from the JSON Schema tree -4. Removes pruned fields from the `required` list + router = APIRouter(route_class=ZCoreAPIRoute) -This means no deep-copy or manual `prune_json_schema` calls are needed β€” the pruning is handled natively by Pydantic V2's schema generation pipeline. + @router.post("/products", openapi_extra={"expose_schema": True}) + async def create(data: ProductCreate, service: ProductService = Inject(ProductService)): + return await service.create(data) + + # GET /products?schema=true β†’ Returns JSON-Schema with restricted fields pruned + # POST /products β†’ Response includes Vary: Authorization, Cookie if pruning occurred + +=== "Standard FastAPI Routing" + :::python + from fastapi import APIRouter + + router = APIRouter() + + @router.post("/products") + async def create(data: ProductCreate, service: ProductService = Depends(get_service)): + # No schema introspection + # No Vary header management + # Plain JSONResponse always used + return await service.create(data) --- -## πŸ›‘οΈ Cache Safety (The `Vary` Header) +## Execution Flow -When an API response is "tailored" (pruned) for a specific user's permissions, intermediate cachesβ€”like **Cloudflare**, **Nginx**, or even a browserβ€”can become a liability. They might cache a "restricted" version of a product and serve it to an admin, or vice versa. +

+ Execution Flow +

-To prevent this data leakage, `ZCoreAPIRoute` automatically manages the `Vary` HTTP header. +--- -| Header Element | Why ZCore adds it | -| :--- | :--- | -| **`Authorization`** | Tells proxies the content changes based on the user's token. | -| **`Cookie`** | Ensures session-based differences are respected by the cache. | +## Boundaries & Integration -```python -# Internal logic: protecting the cache boundary -if hidden_fields: - response.headers["Vary"] = "Authorization, Cookie" -``` +ZCoreAPIRoute works transparently with the rest of the ZCore ecosystem. -!!! info "πŸ›‘οΈ Engineered for Privacy" - By signaling the `Vary` header, ZCore ensures that your infrastructure (CDNs and Proxies) understands that the "same" URL can yield different data depending on who is asking. +* **BaseRouter Compatible:** The `BaseRouter` class in `web/base_router.py` uses `ZCoreAPIRoute` as its default `route_class`. Any scaffolded endpoint automatically inherits schema exposure and Vary header management. +* **Response Class Preservation:** When initializing, `ZCoreAPIRoute` inspects the current `response_class`. If it is the default `JSONResponse`, it swaps in `ZCoreJSONResponse` (which uses ZCore's unified JSON encoder). Custom response classes are left untouched. +* **Manual Endpoints:** Functional endpoints registered on a standard `APIRouter` are unaffected. Only routes using `ZCoreAPIRoute` (or a `BaseRouter` subclass) participate in the security-aware lifecycle. --- -## πŸ› οΈ Schema Resolution Helpers +## Under-the-Hood Spec -To make schema exposure work automatically, ZCore uses internal "discovery" helpers. These allow the framework to find your Pydantic models even when they are wrapped in complex generic types like `List[ProductResponse]` or `Optional[UserResponse]`. +### 1. Response Class Preservation Logic -| Helper | Responsibility | -| :--- | :--- | -| **`find_input_schema`** | Locates the `Zchema`/`BaseModel` used in the request body (for POST/PUT). | -| **`find_output_schema`** | Traverses generic containers to find the core "Response" model. | +Inside `ZCoreAPIRoute.__init__`, the constructor checks the inherited `response_class` [web/api_router.py]. If the class resolves to the standard FastAPI `JSONResponse`, it is replaced with `ZCoreJSONResponse`. Any custom subclass (e.g., `ORJSONResponse`) is preserved, ensuring third-party serializers are never silently overridden. ---- +### 2. Vary Header Append on Context-Pruned Responses + +After the route handler completes, `custom_route_handler` inspects `get_restricted_fields()` [web/api_router.py]. If the set is non-empty and the response is JSON, the method appends `Authorization` and `Cookie` to the existing `Vary` header. Duplicates are eliminated by lower-casing existing values, preventing header bloat on repeated requests. + +### 3. Schema Extraction via Dependency Introspection -## πŸ’‘ Engineering Insights +`find_input_schema` iterates over `dependant.body_params` to locate the body Pydantic model [web/api_router.py]. `find_output_schema` recursively unpacks generic origin types (e.g., `ResponseWrapper[ProductOut]`) to extract the inner model. This allows `ZCoreAPIRoute` to resolve the effective schema even when wrapped in generic containers. -!!! tip "πŸ’‘ Pruning Is Native Now" - Previously, ZCore used a `prune_json_schema` utility that deep-copied and manually traversed dictionary trees. With `Zchema`, the pruning is handled by Pydantic V2's native `__get_pydantic_json_schema__` hook β€” no deep-copying, no manual recursion, and full compatibility with the Pydantic ecosystem. +### 4. Request Body Caching -!!! warning "πŸ›‘οΈ Performance Note" - Schema pruning via `Zchema` is highly optimized and only performed when a client explicitly requests `?schema=true`. Standard API calls bypass the schema pruning logic to maintain high throughput. +`ZCoreRequest` overrides `body()` to cache the byte stream on first read [web/api_router.py]. This prevents downstream middlewares or validators from consuming the stream and leaving subsequent readers with an empty payload. -!!! note "🧠 Automatic Integration" - When you use `BaseRouter`, all these features are enabled by default. You don't need to manually configure `APIRoute` classes; the framework orchestrates the wiring for you. \ No newline at end of file +!!! info "Schema Exposure Restriction" + Schema introspection (`?schema=true`) is only enabled on routes with `openapi_extra={"expose_schema": True}`. This prevents accidental leakage of internal model structures on endpoints that do not opt in. \ No newline at end of file diff --git a/docs/features/security/context.md b/docs/features/security/context.md index a2339c3..85e5cff 100644 --- a/docs/features/security/context.md +++ b/docs/features/security/context.md @@ -1,105 +1,85 @@ -# 🧠 Thread-Safe Context Management +# Thread-Safe Execution Context -ZCore provides a modest but essential system for managing request and user states across asynchronous boundaries using Python's `contextvars` library. This ensures that authenticated user IDs and security restrictions are tracked safely across coroutines, preventing "state leakage" between concurrent requests. +Manage authenticated identities and data-masking policies across asynchronous boundaries with zero state pollution. --- -## πŸ“ Technical Architecture - -The context management system relies on thread-safe and coroutine-safe variables. These act like "global variables" that are private to each specific execution path (a single HTTP request or a background task). - -* πŸ†” **`_current_user_id`**: Stores the unique identifier (UUID) of the authenticated user. -* πŸ›‘οΈ **`_restricted_fields`**: Stores an immutable set of fields that should be hidden or blocked for the current context. - -```mermaid -graph TD - Request[Incoming Request] -->|Authenticate| SetUser[set_current_user_id] - Request -->|Load Permissions| SetFields[set_restricted_fields] - - subgraph Storage [Safe Context Isolation] - SetUser -->|Normalization| VarUser[(_current_user_id)] - SetFields -->|Immutability| VarFields[(_restricted_fields)] - end - - VarUser -->|Auto-Propagate| Tasks[Async Tasks / Coroutines] - VarFields -->|Auto-Propagate| Tasks - - Tasks -->|Finish| Reset[Reset Variables via Tokens] -``` +
+
+ Type
+ State Management +
+
+ Status
+ Core Core +
+
+ Underlying Tech
+ contextvars +
+
+ +## The Challenge +In a high-concurrency asynchronous application, tracking the "current user" or active security policies is notoriously difficult. Developers often face: +1. **Argument Drilling:** Passing a `user` object through five layers of service and repository calls just to log an ID or filter a query. +2. **State Leakage:** Using standard global variables or thread-locals, which can leak data between concurrent requests in an `async/await` environment. +3. **Inconsistent Validation:** Manually checking if a user ID is a valid UUID or if a list of restricted fields is mutable, leading to runtime errors in deep architectural layers. + +## The ZCore Elegance +ZCore utilizes Python's `contextvars` to provide a coroutine-safe execution context. The `request_context` manager allows you to bind a user identity and a set of data-masking restrictions at the start of a request. These variables are then globally accessible to any downstream function (like a Search Engine or Repository) without being explicitly passed as arguments. + +=== "ZCore Context Binding" + :::python + from zcore.context import request_context, get_current_user_id + + # 1. Bind context in a middleware or entry point + with request_context(user_id="...", fields=["products.cost"]): + # 2. Anywhere in your deep service/repository logic: + uid = get_current_user_id() + # uid is resolved automatically from the coroutine stack + +=== "FastAPI Manual Passing" + :::python + # You must pass the user through every single function + async def get_products(user_id: uuid.UUID, restricted: list[str]): + return await repo.find_all(user_id, restricted) + + async def find_all(user_id: uuid.UUID, restricted: list[str]): + # This becomes a maintenance nightmare in large apps + query = select(Product).where(Product.owner_id == user_id) + # ... --- -## πŸ›‘οΈ Safe Context Setters +## Boundaries & Integration +ZCore Context is a low-level utility that bridges the gap between Web and Infrastructure layers. -To ensure data integrity, ZCore's context setters perform two modest but important transformations: - -1. **UUID Normalization**: When you set a user ID, ZCore automatically converts string inputs into proper `uuid.UUID` objects. This prevents "type mismatch" errors in your database queries later. -2. **Immutability Enforcement**: The `set_restricted_fields` helper converts lists or sets into a `frozenset`. This ensures that once a request starts, the list of restricted fields cannot be accidentally modified by other parts of the application. +* **Native ContextVars:** ZCore uses standard `contextvars.ContextVar`. This means it integrates perfectly with standard Python logging (via `structlog`) and any other library that supports the `contextvars` protocol. +* **Scoped Middleware:** The `ScopedDependencyMiddleware` automatically wraps every HTTP request in an empty context, ensuring a clean slate for every transaction. +* **Bypass:** If you prefer explicit dependency injection, you can continue to pass `user` objects manually. ZCore's context is an *additional* capability, not a restrictive requirement. --- -## πŸ“‹ Restricted Fields Syntax - -Restricted fields are defined using the `{db_name}.{field}` syntax. The `db_name` corresponds to the `__db_name__` class attribute defined on your `Zchema` subclass: - -```python -# Setting restricted fields with domain-specific namespaces -set_restricted_fields({ - "user.email", # Restricts the "email" field in the "user" domain - "product.price", # Restricts the "price" field in the "product" domain - "employee.salary" # Restricts the "salary" field in the "employee" domain -}) -``` - -!!! tip "πŸ’‘ Why `{db_name}.{field}`?" - The previous `resource.` prefix syntax caused namespace collisions across different domains. The new `{db_name}.{field}` syntax ensures strict domain boundary isolation, preventing fields in the `billing` module from accidentally affecting fields in the `crm` module. +## The Execution Flow -### πŸ”„ Backward Compatibility +

+ The Execution Flow +

-The old `resource.` prefix syntax is still supported for backward compatibility. ZCore normalizes `resource.` paths by stripping the `resource.` prefix before resolving them against `__db_name__`. However, migrating to the new `{db_name}.{field}` syntax is strongly recommended. --- -## πŸ“‹ Context Managers Reference - -ZCore provides three context managers to safely handle states. They use "Tokens" to ensure that when a block of code finishes, the context is restored to exactly how it was before. - -| Context Manager | Parameters | Practical Use Case | -| :--- | :--- | :--- | -| **`user_context`** | `user_id` | Temporarily "impersonating" a user during a background job. | -| **`restricted_fields_context`** | `fields` | Overriding field visibility for a specific logic block. | -| **`request_context`** | `user_id`, `fields` | **Primary:** Used by middlewares to initialize the entire request state. | - ---- - -## πŸ’» Practical Usage - -Using these context managers ensures that you don't have to pass `user_id` as a parameter through every single function in your application. - -```python -from zcore.context import request_context, get_current_user_id - -# 1. Define the context boundary -with request_context( - user_id="d3b07384-d113-495d-9c31-72f1aa11a43a", - fields=["employee.salary", "employee.internal_notes"] -): - # 2. Any function called inside here can access the context - uid = get_current_user_id() - print(f"Executing task for user: {uid}") - -# 3. Once the 'with' block exits, the context is automatically cleared. -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. Token-Based Restoration Safety +ZCore context managers (`user_context`, `request_context`) utilize the `contextvars.Token` system [context/context.py]. When a context is set, a token is generated. Upon exiting the block, the state is **explicitly reset** to its previous value using this token. This guarantees that even if a request crashes or a task is recycled, the security context never "bleeds" into the next execution. -!!! tip "πŸ’‘ Asynchronous Task Safety" - Because ZCore uses `contextvars`, Python automatically "copies" these values to any nested async tasks (like those created with `asyncio.create_task`). This means your background operations stay just as secure as your main request thread. +### 2. Immutable Restriction Sets +When you set `restricted_fields`, ZCore automatically converts the input into a `frozenset` [context/context.py]. This ensures that once fields are restricted for a request, they cannot be modified by downstream services, preventing accidental or malicious "unmasking" of sensitive data during a complex transaction. -!!! info "πŸ›‘οΈ Middleware Integration" - In a standard ZCore project, the `RequestLogMiddleware` and `ScopedDependencyMiddleware` use these context tools to ensure that your log entries always include the correct `request_id` and `user_id` automatically. +### 3. Immediate UUID Integrity +The `set_current_user_id` function performs immediate type coercion [context/context.py]. If a string is provided, it is validated and converted to a `uuid.UUID` instance instantly. This ensures that downstream database queries never fail due to string-to-UUID comparison mismatches, moving the validation error to the very boundary of the request. -!!! note "🧠 Manual Overrides" - While ZCore manages most context automatically, you can use `user_context(None)` to explicitly run a block of code as an "Anonymous" user, which is useful for public-facing system tasks. \ No newline at end of file +!!! info "Security Note" + Context variables are isolated per asynchronous task. If you spawn a background task (e.g., `asyncio.create_task`), you must manually re-bind the context if the task requires the user's identity. \ No newline at end of file diff --git a/docs/features/security/exceptions.md b/docs/features/security/exceptions.md index 5fd27ae..4a1872e 100644 --- a/docs/features/security/exceptions.md +++ b/docs/features/security/exceptions.md @@ -1,84 +1,110 @@ -# πŸ›‘οΈ Exception Handling +# Centralized Exception Handler -ZCore manages errors with a modest, structured approach. Instead of letting raw Python errors crash your request or leak internal details, ZCore catches domain-specific exceptions and transforms them into standardized, client-friendly JSON responses. +Map domain-level business failures into a standardized, structured API error envelope with zero boilerplate in your route handlers. --- -## πŸ“‹ Custom Exception Hierarchy +
+
+ Type
+ Error Handling +
+
+ Status
+ Security Integration +
+
+ Underlying Tech
+ FastAPI Exception Handlers +
+
+ +## The Challenge + +Standard FastAPI error handling often leads to inconsistent API responses: + +1. **Scattered Error Logic:** Each endpoint or service layer manually catches failures and returns ad-hoc dictionaries like `{"detail": "Not found"}` or `{"error": "Invalid data"}`, creating a fragmented API contract for frontend consumers. +2. **Missing Diagnostic Context:** Internal details (exception type, metadata payload) are lost when converting to HTTP responses, making debugging production incidents harder. +3. **Inconsistent HTTP Codes:** The same logical error (e.g., "entity not found") might return 404 in one endpoint, 500 in another, and 400 in a third, depending on where the exception was raised. + +## The ZCore Elegance + +ZCore provides a thin `AppException` hierarchy that maps directly to HTTP status codes, plus a single global handler that catches them all and formats them into the standard `ResponseWrapper` envelope. Services and repositories raise typed exceptions without any HTTP awareness. + +=== "ZCore Typed Exceptions" + :::python + from zcore.exceptions import EntityNotFound, ValidationError, ForbiddenError + + class PaymentService(BaseService): + async def process(self, payment_id: uuid.UUID): + payment = await self.repository.get(payment_id) + if not payment: + raise EntityNotFound("Payment record not found") + if not payment.is_approved: + raise ForbiddenError("Insufficient permissions to process") + if payment.amount <= 0: + raise ValidationError("Payment amount must be positive") + return payment + + # FastAPI automatically returns: + # 404 β†’ {"success": false, "message": "Payment record not found", "meta": {"error_type": "EntityNotFound", "payload": null}} + +=== "Standard FastAPI Error Handling" + :::python + from fastapi import HTTPException + + class PaymentService: + async def process(self, payment_id: uuid.UUID): + payment = await self.repository.get(payment_id) + if not payment: + raise HTTPException(status_code=404, detail="Payment not found") + if not payment.is_approved: + raise HTTPException(status_code=403, detail="Forbidden") + if payment.amount <= 0: + raise HTTPException(status_code=400, detail="Invalid amount") + return payment + + # No standard envelope across endpoints + # No automatic error type classification in response + # Manual HTTP code management scattered through business logic -The framework provides a dedicated exception hierarchy where every error inherits from `AppException`. This design allows you to map business failures directly to standard HTTP status codes. +--- -| Exception Class | HTTP Code | Practical Use Case | -| :--- | :--- | :--- | -| πŸ›‘ **`AppException`** | `500` | Generic fallback for unhandled internal failures. | -| πŸ” **`EntityNotFound`** | `404` | When a specific ID or resource does not exist in the DB. | -| β™Š **`DuplicateEntity`** | `409` | When a unique constraint (like a duplicate email) is violated. | -| πŸ”‘ **`AuthError`** | `401` | When credentials are missing, invalid, or expired. | -| 🚫 **`ForbiddenError`** | `403` | When a user is logged in but lacks required permissions. | -| ⚠️ **`ValidationError`** | `400` | When business rules are violated (e.g., "Price cannot be zero"). | +## Boundaries & Integration ---- +The exception handler is a lightweight, opt-in component registered at the application level. -## πŸ› οΈ Central Exception Handler - -ZCore utilizes a global handler (`app_exception_handler`) to intercept any `AppException` raised within your services, repositories, or routers. This ensures that your API remains predictable and never returns a "Broken HTML" page for an error. - -```mermaid -graph TD - Error[AppException Raised] -->|Intercept| Log[Write Warning to Logs] - Log -->|Analyze Context| Envelope[Prepare ResponseWrapper] - Envelope -->|Metadata| Payload[Include error_type & custom payload] - Payload -->|Serialize| JSON[Return Standardized JSON Response] -``` - -### πŸͺ΅ Transparent Logging -Whenever an exception is caught, ZCore logs a diagnostic warning. This allows you to monitor failures in production without exposing sensitive tracebacks to the end-user. - -```python -# Standardized Error Envelope returned to the client: -{ - "success": false, - "message": "Insufficient stock available.", - "data": null, - "meta": { - "error_type": "ValidationError", - "payload": { "available": 2, "requested": 10 } - } -} -``` +* **FastAPI Standard:** The `app_exception_handler` is a standard FastAPI exception handler registered via `app.add_exception_handler(AppException, app_exception_handler)`. It works with any FastAPI application, including those not using other ZCore components. +* **Payload Customization:** Each exception can carry an optional `payload` dictionary. The handler includes this payload in the response `meta` object, enabling structured error details (e.g., field-level validation errors or conflict resource identifiers). +* **Third-Party Compatibility:** Since the handler only catches `AppException` subclasses, standard Python exceptions (e.g., `ValueError`, `KeyError`) and FastAPI's built-in `HTTPException` are unaffected and handled by their default mechanisms. --- -## πŸ’» Practical Usage +## Under-the-Hood Spec -We suggest raising these exceptions inside your **Service Layer** to communicate business failures clearly. +### 1. Status Code Mapping via Class Attribute -```python -from zcore.exceptions.base import ValidationError, EntityNotFound +Each `AppException` subclass defines `status_code` as a class-level integer [exceptions/base.py]. `EntityNotFound` maps to `404`, `ValidationError` to `400`, `AuthError` to `401`, `ForbiddenError` to `403`, and `DuplicateEntity` to `409`. Custom exceptions inherit from `AppException` and can override `status_code` to any valid HTTP status. -async def process_order(item_id: str, quantity: int): - # 1. Check if resource exists - item = await repository.get(item_id) - if not item: - raise EntityNotFound(message=f"Product '{item_id}' was not found.") - - # 2. Validate business constraints - if item.stock < quantity: - raise ValidationError( - message="Not enough items in stock.", - payload={"stock": item.stock, "requested": quantity} - ) -``` +### 2. Structured Logging Capture ---- +The `app_exception_handler` logs every exception through `structlog` before constructing the response [exceptions/handlers.py]. The log entry includes `type`, `status_code`, `message`, `payload`, `path`, and `method`β€”providing a complete audit trail for debugging without exposing sensitive internals to the client. -## πŸ’‘ Engineering Insights +### 3. Standard API Error Envelope -!!! tip "πŸ’‘ Structural vs. Business Validation" - Use **Pydantic Schemas** for structural validation (e.g., "Is this an email?"). Use **`ValidationError`** inside your Service for business validation (e.g., "Is this email already taken?"). This keeps your layers clean and specialized. +The handler constructs a `ResponseWrapper[None]` with `success=False` [exceptions/handlers.py]. The `message` field carries the human-readable error string, while `meta.error_type` stores the exact exception class name (e.g., `"EntityNotFound"`) for programmatic consumption. The `meta.payload` field is included only if the exception was raised with contextual metadata. -!!! info "πŸ›‘οΈ Payload Flexibility" - The `payload` parameter in `AppException` is a modest dictionary. You can use it to return specific field errors, error codes, or helpful hints to the frontend developer, all wrapped inside the `meta` field of the response. +!!! info "Registration in main.py" + The exception handler must be explicitly registered during application bootstrap: + ```python + from zcore.exceptions import AppException, app_exception_handler + app.add_exception_handler(AppException, app_exception_handler) + ``` -!!! warning "🧹 Fail-Fast Philosophy" - ZCore encourages "Failing Fast." As soon as a business rule is violated, raise the appropriate exception. The `UnitOfWork` will automatically detect this and rollback any pending database changes, ensuring your data stays consistent. +!!! tip "Extending the Hierarchy" + To add a custom error type, subclass `AppException` and set `status_code`: + ```python + class RateLimitExceeded(AppException): + status_code = 429 + ``` + The existing handler automatically picks up the new type at runtime. \ No newline at end of file diff --git a/docs/features/security/security.md b/docs/features/security/security.md index 644c00e..0f107e3 100644 --- a/docs/features/security/security.md +++ b/docs/features/security/security.md @@ -1,103 +1,97 @@ -# πŸ›‘οΈ Cryptographic Protocols & Security +# Cryptographic Foundations -ZCore takes a modest and practical approach to security. It comes equipped with industry-standard protocols for password protection and identity management, ensuring your application follows modern safety guidelines without requiring complex manual configuration. +Secure user credentials with Argon2id and orchestrate identity via hardened, protocol-agnostic JWT signaling. --- -## πŸ”’ Argon2id Password Hashing - -For password security, ZCore utilizes **Argon2id**. This is currently considered a modern standard because it is specifically designed to be resistant to both brute-force attacks and highly specialized GPU-based cracking. - -### βš™οΈ Default Security Parameters -The Argon2 engine is pre-configured with high-security defaults. These values ensure a strong defense while remaining efficient for modern server hardware: - -| Parameter | Default Value | Purpose | -| :--- | :--- | :--- | -| **Memory Cost** | `65536` (64 MB) | Forces the attacker to use significant RAM for each guess. | -| **Time Cost** | `3` Iterations | Determines the number of passes over the memory. | -| **Parallelism** | `4` Threads | Utilizes multi-core processing for hashing. | - -### πŸ›‘οΈ Side-Channel Protection -To prevent "Timing Attacks" (where an attacker guesses a password by measuring how long the server takes to respond), ZCore's verification function always returns a boolean outcome safely. It swallows internal cryptographic errors to ensure the execution time remains consistent. - -```python -# Internal safety logic -try: - return ph.verify(hashed_password, plain_password) -except (VerifyMismatchError, InvalidHashError): - return False # Consistent failure response -``` +
+
+ Type
+ Security Infrastructure +
+
+ Status
+ Core Core +
+
+ Underlying Tech
+ Argon2id / PyJWT +
+
+ +## The Challenge +Implementing robust security in FastAPI often suffers from "Infrastructure Drift": +1. **Stale Hashing:** Many projects still use BCrypt or PBKDF2, which are increasingly vulnerable to GPU-accelerated side-channel attacks. +2. **Secret Mismanagement:** Hardcoding secrets or failing to rotate them, often running production apps with default "example" keys. +3. **Rigid Token Logic:** Difficulty switching from Symmetric (HS256) to Asymmetric (RS256) signing as the architecture moves toward microservices or OIDC. +4. **Implicit Expiry:** Manual timestamp math for `exp` claims that lead to timezone bugs or permanent tokens. + +## The ZCore Elegance +ZCore provides a hardened security chassis. It defaults to **Argon2id** (the winner of the Password Hashing Competition) with configurable memory costs. The JWT engine is "Signature-Aware," automatically detecting whether to use Symmetric or Asymmetric keys based on your configuration, while enforcing a mandatory "Production Secret Guard" that prevents unsafe deployments. + +=== "ZCore Hardened Security" + :::python + from zcore.security import get_password_hash, create_token, verify_password + + # 1. High-Entropy Hashing (Argon2id) + hashed = get_password_hash("user_password") + is_valid = verify_password("user_password", hashed) + + # 2. Protocol-Agnostic Token Creation + # Automatically uses RSA/ECDSA if keys are configured + token = create_token(data={"sub": str(user.id), "scopes": ["admin"]}) + +=== "FastAPI Manual Security" + :::python + # Requires manual configuration of Passlib and PyJWT + from passlib.context import CryptContext + import jwt + + pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + + def create_token(data: dict): + to_encode = data.copy() + # Manual expiry math + expire = datetime.utcnow() + timedelta(minutes=30) + to_encode.update({"exp": expire}) + # Manual algorithm and key management + return jwt.encode(to_encode, SECRET_KEY, algorithm="HS256") --- -## πŸ”‘ JSON Web Tokens (JWT) +## The Token Lifecycle -ZCore provides a modest wrapper for managing JWTs, supporting two primary modes of operation depending on your infrastructure needs: +

+ The Token Lifecycle +

-1. **Symmetric (HMAC):** A simple approach using a shared `SECRET_KEY`. Ideal for smaller, single-service applications. -2. **Asymmetric (RSA/ECDSA):** A more advanced approach using a Private/Public key pair. This allows other services to verify tokens using your public key without knowing your private signing key. --- -## 🚨 The Production "Safety Shield" +## Boundaries & Integration +ZCore manages the cryptographic "heavy lifting" while remaining compatible with standard identity providers. -A common engineering mistake is accidentally deploying an application to production with a default "test" secret key. ZCore implements a **Fatal Startup Guard** to prevent this. - -If the framework detects it is running in a `production` environment while still using the insecure default key, it will immediately abort the startup sequence. - -```mermaid -graph TD - Start[App Bootstrapping] --> EnvCheck{Is Environment 'Production'?} - EnvCheck -->|No| Run[Continue Startup] - EnvCheck -->|Yes| KeyCheck{Is SECRET_KEY default?} - KeyCheck -->|No| Run - KeyCheck -->|Yes| Abort[🚨 FATAL SECURITY VIOLATION: Abort Startup] -``` +* **OAuth2 Integration:** ZCore's `create_token` output is a standard JWT string, fully compatible with FastAPI's `OAuth2PasswordBearer` and `Security` scopes. +* **User Independence:** The security utilities do not require a Database connection. You can use them to secure machine-to-machine (M2M) communication or external microservices. +* **Bypass:** If your organization requires a different hashing algorithm (e.g., Scrypt), you can ignore the `hashing.py` utility. ZCore's higher-level components do not force a specific hash format on your `User` model. --- -## πŸ’» Practical Usage - -### 1. Handling Passwords -We suggest hashing passwords at the point of registration and verifying them only during the login phase. - -```python -from zcore.security import get_password_hash, verify_password - -# πŸ“ During User Registration: -hashed_pw = get_password_hash("user-password-123") - -# πŸ” During User Login: -if verify_password("user-password-123", hashed_pw): - print("βœ… Access Granted") -``` - -### 2. Issuing Tokens -Tokens can carry custom "Claims" (data) such as the user ID or specific permission scopes. - -```python -from zcore.security import create_token, decode_token - -# πŸ“€ Create a token (Default expiry: 30 minutes) -token = create_token(data={"sub": "user_id_123", "role": "admin"}) - -# πŸ“₯ Decode and validate a token -try: - payload = decode_token(token) - user_id = payload.get("sub") -except AuthError as e: - print(f"❌ Validation failed: {e.message}") -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The Production Secret Guard +To prevent critical security oversights, ZCore’s JWT key resolver (`_get_signing_keys`) includes a fatal check [security/jwt.py]. If `ENVIRONMENT` is "production" and `SECRET_KEY` is set to the framework's insecure default value, the system raises a `RuntimeError` and **aborts application startup**. -!!! tip "πŸ’‘ Why Argon2id?" - Unlike older algorithms (like MD5 or SHA1), Argon2id is "memory-hard." This makes it extremely expensive for attackers to build specialized hardware to crack your users' passwords. +### 2. Argon2id Configuration +ZCore initializes the `PasswordHasher` with parameters optimized for modern server hardware [security/hashing.py]: +- **Memory Cost:** 64 MB (`65536`) +- **Time Cost:** 3 Iterations +- **Parallelism:** 4 Threads +These parameters ensure high resistance to specialized hardware (ASIC/GPU) cracking while maintaining sub-100ms verification times for users. -!!! info "πŸ›‘οΈ Token Expiration" - ZCore's `decode_token` automatically checks the `exp` (expiration) claim. If the token is even one second past its limit, it will raise an `AuthError`. You don't need to manually check timestamps. +### 3. Dynamic Asymmetric Detection +The JWT utility is designed for scale [security/jwt.py]. If `JWT_PRIVATE_KEY` and `JWT_PUBLIC_KEY` are detected in the settings, the engine automatically switches to Asymmetric signing (e.g., `RS256`). This allows the framework to sign tokens in the Auth service that can be verified by downstream services using only the Public Key, without sharing the master secret. -!!! warning "πŸ”‘ Asymmetric Setup" - To use Asymmetric signing, you must provide `JWT_PRIVATE_KEY` and `JWT_PUBLIC_KEY` in your settings. If these are missing, ZCore gracefully falls back to Symmetric (HMAC) signing using your `SECRET_KEY`. +!!! danger "Security Warning" + Never store your `SECRET_KEY` or `JWT_PRIVATE_KEY` in version control. Always use environment variables or a secure secret manager (like Vault or AWS Secrets Manager) in production. \ No newline at end of file diff --git a/docs/features/security/zchema.md b/docs/features/security/zchema.md index e8b6c5e..5838850 100644 --- a/docs/features/security/zchema.md +++ b/docs/features/security/zchema.md @@ -1,154 +1,115 @@ -# πŸ›‘οΈ Unified Schema Pruning & Security (`Zchema`) +# Zchema Security & Schema Pruning -ZCore provides a modest but powerful security layer that lives directly inside your Pydantic schemas. Instead of relying on web-layer middlewares or a separate `ResponseProjector` to prune data after rendering, ZCore introduces the `Zchema` base class β€” a drop-in replacement for Pydantic's `BaseModel` that automatically secures your data at three distinct tiers. +Unify input validation, response serialization, and dynamic JSON-Schema generation into a single domain-aware base class that silently enforces data-masking policies. --- -## 🧱 The `Zchema` Base Class +
+
+ Type
+ Schema Security Layer +
+
+ Status
+ Security Core +
+
+ Underlying Tech
+ Pydantic V2 / contextvars +
+
-`Zchema` extends Pydantic V2's native hooks to inject security directly into the schema lifecycle. Every schema you define should inherit from `Zchema` instead of `BaseModel`: +## The Challenge -```python -from zcore import Zchema +In a typical FastAPI application, security schema management suffers from three distinct failure modes: -class UserBase(Zchema): - __db_name__ = "user" - email: str - salary: float -``` +1. **Schema Fragmentation:** To prevent data leakage, developers define multiple Pydantic models per entity (e.g., `ProductCreate`, `ProductOutPublic`, `ProductOutAdmin`, `ProductUpdate`). A single new restricted field must be added to every variant, leading to copy-paste errors. +2. **Mass Assignment Vulnerabilities:** Input schemas that accept all model fields allow a malicious user to send unexpected fields (e.g., `is_admin=true`) that get written directly to the database. +3. **Context-Blind Serialization:** Response serialization is static. A service returns the same fields regardless of who is making the request, forcing every endpoint to manually check permissions before building the response. -### 🏷️ The `__db_name__` Attribute +## The ZCore Elegance -The `__db_name__` class attribute binds the schema to a specific database domain. This is the foundation of ZCore's domain-isolated security model: +`Zchema` is a drop-in replacement for `pydantic.BaseModel` that is domain-aware and context-sensitive. You define a single schema per entity, annotate it with `__model__` to declare its database domain, and Zchema automatically prunes restricted fields during validation, serialization, and JSON-Schema generationβ€”all based on the active thread-safe context. -* **Namespace Isolation:** Restricted fields are defined as `{db_name}.{field}` (e.g., `user.email`, `order.total`), preventing collisions between modules. -* **Scoped Resolution:** The security layer uses `__db_name__` to determine which restricted paths apply to which schema, ignoring fields from unrelated domains. -* **Backward Compatibility:** The old `resource.` prefix syntax is still supported as a fallback, but the new `{db_name}.` syntax is the recommended approach. +=== "ZCore Unified Zchema" + :::python + from zcore.web.projection import Zchema ---- - -## πŸ” Three-Tier Security Model + class ProductBase(Zchema): + __model__ = "products" # Domain binding + name: str + price: float + secret_cost: float | None = None # Restricted when context says so -`Zchema` automatically protects your data across three independent tiers, all without any web-layer middleware configuration: + # Inside a request where "products.secret_cost" is restricted: + schema = ProductBase(name="X", price=10, secret_cost=100) + # schema.model_dump() β†’ {"name": "X", "price": 10} (secret_cost pruned) -```mermaid -graph BT - subgraph Tier_1[πŸ” Tier 1: Dynamic Schema Generation] - A1[Client Request: ?schema=true] --> B1[__get_pydantic_json_schema__] - B1 --> C1[Resolve __db_name__ context] - C1 --> D1[Prune restricted fields from JSON Schema] - D1 --> E1[Return pruned schema to client] - end + # GET /products?schema=true returns JSON-Schema WITHOUT secret_cost - subgraph Tier_2[πŸ“₯ Tier 2: Input Validation] - A2[Malicious Payload with restricted fields] --> B2[model_validator filter_restricted_inputs] - B2 --> C2[Silently strip unauthorized fields] - C2 --> D2[Pass clean payload to business logic] - end +=== "Standard Pydantic Multi-Model" + :::python + from pydantic import BaseModel - subgraph Tier_3[πŸ“€ Tier 3: Response Pruning] - A3[Outgoing serialized data] --> B3[model_serializer secure_serializer] - B3 --> C3[Strip restricted fields recursively] - C3 --> D3[Return sanitized JSON to client] - end -``` + class ProductCreate(BaseModel): + name: str + price: float + secret_cost: float | None = None -### πŸ” Tier 1: Dynamic Schema Generation + class ProductOut(BaseModel): + name: str + price: float + # secret_cost is missingβ€”but so is any admin-only field -When a client appends `?schema=true` to an endpoint URL, ZCore dynamically generates the JSON Schema for the target model. The `Zchema.__get_pydantic_json_schema__` hook intercepts this process and removes any properties that match the active context's restricted fields, ensuring that even the schema metadata doesn't leak information about hidden fields. + # Must maintain N models per entity + # Each new field requires updates across ALL variants + # No automatic context-driven pruning -### πŸ“₯ Tier 2: Input Validation (Mass Assignment Prevention) - -A malicious user could try to send restricted fields (e.g., `is_admin: true`) in a POST or PUT request body. The `model_validator` (`filter_restricted_inputs`) silently strips these fields before they ever reach your service layer, effectively preventing Mass Assignment attacks without any additional code. +--- -### πŸ“€ Tier 3: Response Pruning +## The Pruning Pipeline -During serialization, the `model_serializer` (`secure_serializer`) intercepts the output and recursively removes any keys that match the active context's restricted fields. Unlike the old `ResponseProjector` approach β€” which operated on raw JSON after rendering β€” this happens **natively within the model**, making it compatible with any serialization pipeline. +

+ The Pruning Pipeline +

--- -## πŸ’» Practical Usage +## Boundaries & Integration -### Defining Schemas with Domain Binding +Zchema is the backbone of ZCore's data-masking strategy and integrates seamlessly with the broader ecosystem. -```python -from zcore import Zchema -from pydantic import Field, ConfigDict -import uuid +* **ResponseWrapper Inheritance:** `ResponseWrapper` inherits from `Zchema`, meaning all API responses automatically benefit from context-driven serialization pruning. +* **BaseRouter Schema Resolution:** The `BaseRouter` uses `create_schema`, `update_schema`, and `schema_out`β€”all expected to be `Zchema` subclasses. The router's `ZCoreAPIRoute` invokes `model_json_schema()` on these classes, which triggers the customized schema generation path. +* **Context Decoupling:** Zchema never directly inspects HTTP headers or tokens. It reads restricted field paths from the thread-safe context (`get_restricted_fields()`), which is populated by upstream middleware or authentication logic. -class EmployeeResponse(Zchema): - __db_name__ = "employee" - id: uuid.UUID - name: str - salary: float # Will be pruned for non-admin users - bank_account_number: str # Will be pruned for non-admin users +--- - model_config = ConfigDict(from_attributes=True) -``` +## Under-the-Hood Spec -### Setting Restricted Fields in Context +### 1. Domain Mapping via `__model__` -Restricted fields are defined using the `{db_name}.{field}` syntax and bound to the request context via middleware or a dependency: +Each Zchema subclass declares a `__model__` class variable matching its database table or domain name (e.g., `"products"`) [web/projection.py]. When the context holds a restricted path like `"products.secret_cost"`, `_get_relative_restricted_paths` strips the domain prefix and returns `{"secret_cost"}`. A wildcard path `"products"` maps to `{"*"}`, which clears the entire payload. -```python -from zcore.context import set_restricted_fields +### 2. Input Pruning via `model_validator(mode="before")` -# In a security dependency or middleware: -set_restricted_fields({"employee.salary", "employee.bank_account_number"}) -``` +The `filter_restricted_inputs` validator executes before any Pydantic field validation [web/projection.py]. It makes a shallow copy of the input dictionary and calls `_prune_data` to remove any keys whose dotted paths match the restricted set. This prevents mass assignment attacks where a user injects privileged fields into a create or update payload. -ZCore will automatically resolve these paths against the `__db_name__` of any schema being serialized. If the domain matches, the corresponding fields are pruned across all three security tiers. +### 3. Output Pruning via `model_serializer(mode="wrap")` ---- +The `secure_serializer` wraps the default serialization handler [web/projection.py]. After Pydantic produces the standard dictionary, the method applies the same pruning logic to strip restricted fields from the serialized output before returning the response. -## πŸ“¦ Standardized Response Envelope (`ResponseWrapper`) - -The `ResponseWrapper` class itself inherits from `Zchema`, meaning it also participates in the security pruning pipeline: - -```python -from zcore.web.response import ResponseWrapper - -@app.get("/custom-stats") -async def get_stats(): - stats_data = {"active_users": 150, "uptime": "99.9%"} - - # Manually wrap your data in a standardized success response - return ResponseWrapper.success_response( - data=stats_data, - message="System statistics retrieved successfully" - ) -``` - -### πŸ“ Example Response Structure - -```json -{ - "success": true, - "message": "Operation completed successfully", - "data": { - "id": "e4c02f06-d71d-4876-88b0-a3e390c58a62", - "name": "Heavy Duty Widget", - "price": 249.99 - }, - "meta": { - "execution_time_ms": 12.4 - } -} -``` - -### πŸ’‰ Pydantic Generic Support - -The `ResponseWrapper` uses Pydantic **Generics**. This means your API documentation (Swagger/OpenAPI) remains perfectly type-safe. If your endpoint returns a `Product`, the documentation will show `ResponseWrapper[Product]`, making it clear what the `data` field contains. +### 4. Schema Pruning via `__get_pydantic_json_schema__` ---- +Zchema overrides Pydantic V2's JSON-Schema generation hook [web/projection.py]. It calls the standard handler to produce the full schema, then recursively walks the `properties` and `required` lists to remove any fields that are restricted for the current context. This ensures that frontend forms generated from `?schema=true` never include fields the user cannot see or edit. -## πŸ’‘ Engineering Insights +### 5. Recursive Nested Pruning -!!! tip "πŸ’‘ Why `Zchema` Instead of `ResponseProjector`?" - The old `ResponseProjector` operated on raw JSON after rendering, which required a custom `ZCoreJSONResponse` class and introduced an extra deserialization/reserialization cycle. `Zchema` integrates security directly into Pydantic's native hooks β€” the pruning happens once, at the model level, without any web-layer middleware. +The `_prune_data` helper handles nested structures [web/projection.py]. For a restriction like `"orders.items.price"`, it descends into `data["orders"]["items"]` and prunes the `price` key from each dictionary or list element. This enables field-level masking across complex, deeply nested Pydantic models. -!!! info "πŸ›‘οΈ Domain Boundary Isolation" - Because `__db_name__` ties each schema to a specific domain (e.g., `"user"`, `"product"`, `"order"`), there is zero risk of field name collisions between modules. A restricted path like `user.email` will never accidentally prune the `email` field from the `Product` schema. +!!! info "Performance Note" + Zchema restricts input and output fields by value-copying the data dictionary before pruning. This ensures the original model instance is never mutated, preserving data integrity for downstream operations that may bypass the serializer. -!!! note "🧠 Handling Empty Data" - If an endpoint returns `None` (for example, after a successful deletion), the `ResponseWrapper` will still return a success status with `data: null`. This consistency ensures that frontend JSON parsers don't crash when expecting a specific structure. \ No newline at end of file +!!! danger "Security Note" + Because pruning happens at the Pydantic layer, it applies universallyβ€”including in background tasks, CLI commands, or any code path that uses `model_dump()` on a Zchema instance while a restricted context is active. Always reset the context when spawning isolated work. \ No newline at end of file diff --git a/docs/how-to/auth-setup.md b/docs/how-to/auth-setup.md index cb5d505..afc04da 100644 --- a/docs/how-to/auth-setup.md +++ b/docs/how-to/auth-setup.md @@ -1,154 +1,132 @@ -# πŸ” How-To: Enforcing Full JWT Authentication & Dynamic Scopes +# JWT Authentication & Scopes -## ❓ The Problem - -By default, ZCore uses a placeholder stub dependency (`get_current_user_stub`) for route authorization checks. If you attempt to use route-level security guards out of the box, ZCore will raise a `NotImplementedError`. - -To secure your application, you need to implement a concrete token verification flow, bind your user model to ZCore's security requirements, and override the default stub dependency. +Secure your API with Argon2id hashing, flexible JWT signing, and protocol-based authorization guards. --- -## πŸ› οΈ The ZCore Solution - -We suggest implementing a complete authentication flow using ZCore's JWT decoding utilities and registering it as a global FastAPI dependency override. - -```mermaid -graph TD - Request[HTTP Request with Bearer Token] -->|FastAPI Routing| Dep[get_current_user Dependency] - Dep -->|Extract Header| Token{Token Present?} - Token -->|No| AuthErr1[Raise AuthError: Authentication required] - Token -->|Yes| Decode[decode_token] - Decode -->|Expired / Invalid Signature| AuthErr2[Raise AuthError: Token expired / Invalid] - Decode -->|Valid| Query[Query DB / Verify User Status] - Query -->|Checks UserProtocol| ReturnUser[Return concrete User instance] - ReturnUser -->|Binds Context| Route[Execute endpoint with HasScopes Guard] -``` +
+
+ Type
+ Security / Protocol +
+
+ Status
+ Core Core +
+
+ Underlying Tech
+ Argon2id / PyJWT / contextvars +
+
+ +## The Challenge +Implementing security in FastAPI often leads to "Hardcoded Identity" problems. Developers either tightly couple their routes to a specific SQLAlchemy User model or manually parse JWTs in every dependency. Furthermore, many projects still use aging hashing algorithms like BCrypt or, worse, run production environments with default, insecure secret keys found in git history. + +## The ZCore Elegance +ZCore introduces **Protocol-Based Security**. Instead of depending on a concrete model, the security layer depends on a `UserProtocol`. This allows you to use any database (SQL, NoSQL, or External Auth) as long as your user object provides the required attributes. Security is enforced through reusable `BasePermission` classes that integrate seamlessly with `BaseRouter` and the kernel's context. + +=== "ZCore Permission Guards" + :::python + from zcore import BaseRouter, RouteKey + from zcore.security import HasScopes + + class DocumentRouter(BaseRouter[DocCreate, DocUpdate]): + model = Document + # ... standard config ... + + def get_route_dependencies(self, route_key: RouteKey, action: str): + # Automated scope checks based on model actions + # e.g., "documents:delete" + if route_key == RouteKey.DELETE: + return [HasScopes("admin", action)] + + return super().get_route_dependencies(route_key, action) + +=== "FastAPI Manual Auth" + :::python + # Manual verification in every route + @router.delete("/{id}") + async def delete_doc( + id: uuid.UUID, + user: User = Depends(get_current_user) + ): + # 1. Manual active check + if not user.is_active: raise AuthError() + + # 2. Manual scope string management + if "admin" not in user.scopes and "documents:delete" not in user.scopes: + raise ForbiddenError() + + # 3. Manual service call + await service.delete(id) --- -### πŸ“¦ Step 1: Implement the User Protocol +## Implementation Guide -First, create a user class that implements ZCore's `UserProtocol`. The user model must provide `id`, `is_active`, `is_superuser`, and the dynamic property `all_scopes`: +### 1. Define your User Model +Ensure your SQLAlchemy or Pydantic user model implements the `UserProtocol`. ```python -import uuid -from typing import Set -from sqlalchemy import String, Boolean, JSON -from sqlalchemy.orm import Mapped, mapped_column -from zcore.db.setup import Base - class User(Base): - """Database representation matching ZCore's UserProtocol.""" __tablename__ = "users" - - id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) - email: Mapped[str] = mapped_column(String(255), unique=True, index=True) - is_active: Mapped[bool] = mapped_column(Boolean, default=True) - is_superuser: Mapped[bool] = mapped_column(Boolean, default=False) - scopes: Mapped[list[str]] = mapped_column(JSON, default=list) # e.g. ["products:view"] - + id: Mapped[uuid.UUID] = mapped_column(primary_key=True) + is_active: Mapped[bool] = mapped_column(default=True) + is_superuser: Mapped[bool] = mapped_column(default=False) + @property - def all_scopes(self) -> Set[str]: - """Compile and return the dynamic scope set for this user.""" - scopes_set = set(self.scopes) - if self.is_superuser: - # Grant superusers implicit system-wide privileges - scopes_set.add("*") - return scopes_set + def all_scopes(self) -> set[str]: + # Return a set of strings representing user permissions + return {r.name for r in self.roles} ``` ---- - -### πŸ›‘οΈ Step 2: Implement the Security Dependency - -Next, create a security dependency that extracts the Bearer token from the HTTP `Authorization` header, decodes it using ZCore's `decode_token`, and validates the active user: +### 2. Override the Identity Provider +In your `main.py`, you must tell ZCore how to resolve the current user. ZCore provides a `get_current_user_stub` that you must override. ```python -from fastapi import Request, Depends -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from sqlalchemy import select - -from zcore.exceptions.base import AuthError -from zcore.security.jwt import decode_token -from zcore.db.setup import get_db, AsyncSession -from .models import User - -# Initialize standard Bearer token scheme extractor -security_scheme = HTTPBearer(auto_error=False) - -async def get_current_user( - credentials: HTTPAuthorizationCredentials = Depends(security_scheme), - db: AsyncSession = Depends(get_db) -) -> User: - """Validate incoming JWT credentials and return the active user model.""" - if not credentials or credentials.scheme.lower() != "bearer": - raise AuthError(message="Authentication required. Please provide a Bearer token.") - - token = credentials.credentials - # Decode the token (automatically raises AuthError if expired or malformed) - payload = decode_token(token) - user_id_str = payload.get("sub") +from zcore.security import get_current_user_stub, decode_token - if not user_id_str: - raise AuthError(message="Invalid token structure: missing user identifier claims.") - - # Retrieve the user from the database - query = select(User).where(User.id == user_id_str) - result = await db.execute(query) - user = result.scalars().first() - - if not user: - raise AuthError(message="User account associated with this token was not found.") - - if not user.is_active: - raise AuthError(message="User account is deactivated.") +async def my_auth_provider(token: str = Depends(oauth2_scheme)) -> User: + payload = decode_token(token) + user_id = payload.get("sub") + # Fetch user from DB + return await user_service.get(user_id) - return user +# Register the override +app.dependency_overrides[get_current_user_stub] = my_auth_provider ``` --- -### βš™οΈ Step 3: Register the Dependency Override +## The Auth Lifecycle -In your `main.py` entry point, register your concrete dependency as an override for ZCore's authentication stub: +

+ The Auth Lifecycle +

-```python -from fastapi import FastAPI -from zcore.security.dependencies import get_current_user_stub -from .auth import get_current_user -app = FastAPI() +--- + +## Boundaries & Integration +ZCore provides the cryptographic and logic chassis, while you provide the data. -# Register the concrete override in FastAPI's dependency map -app.dependency_overrides[get_current_user_stub] = get_current_user -``` +* **OAuth2 Compatible:** ZCore's JWT utilities work perfectly with FastAPI's `OAuth2PasswordBearer`. +* **Hashing Freedom:** While ZCore provides `get_password_hash` using Argon2id, you can use any library for hashing; ZCore services and repositories accept raw strings. +* **Bypass Identity:** If you have public routes, simply don't add the `HasScopes` dependency. The `request_context` will remain empty (user_id = None), and ZCore's masking logic will treat the request as unauthenticated. --- -### 🚦 Step 4: Secure Your API Endpoints - -With the dependency override in place, you can secure your endpoints using ZCore's `HasScopes` guard. The guard automatically checks if the resolved user possesses the required scopes: +## Under-the-Hood Spec -```python -from fastapi import APIRouter, Depends -from zcore.security.permissions import HasScopes -from zcore.web.response import ResponseWrapper - -router = APIRouter(prefix="/inventory", tags=["Inventory"]) - -# Secure a single endpoint with scope permissions -@router.post( - "/", - dependencies=[Depends(HasScopes("products:create"))], - response_model=ResponseWrapper -) -async def create_inventory_item(): - return ResponseWrapper(message="Item created successfully") -``` +### 1. Fatal Security Violation Guard +The `jwt.py` module contains a hard-coded check [security/jwt.py]. If the `ENVIRONMENT` is set to `production` and the `SECRET_KEY` matches the framework's default fallback string, the application will **abort startup** with a `RuntimeError`. This prevents the most common source of production JWT vulnerabilities. ---- +### 2. Argon2id Hardening +ZCore defaults to Argon2id (the winner of the Password Hashing Competition). The `hashing.py` module configures high-entropy parameters by default: 64MB memory cost and 3 iterations [security/hashing.py]. These parameters can be tuned via `ZCoreCoreSettings` to balance security and latency. -## πŸ’‘ Engineering Insights +### 3. Protocol-Based "Duck Typing" +The `HasScopes` permission does not perform an `isinstance()` check against a database model [security/permissions.py]. Instead, it relies on the `UserProtocol` interface. This means your "User" could be a database record, a cached Redis object, or even a Mock object during testing, as long as it has `id`, `is_active`, and `all_scopes`. -!!! tip "πŸ’‘ Superuser Bypass" - The `HasScopes` validator contains an `allow_superuser` configuration option (enabled by default). If set to `True`, the permission check is bypassed for users where `is_superuser` is `True`, allowing administrators unrestricted access. \ No newline at end of file +!!! success "Security Note" + By utilizing the `user_context` within your identity provider, the authenticated user's ID is available project-wide via `get_current_user_id()` without passing the user object through every function call. \ No newline at end of file diff --git a/docs/how-to/base-router-extensibility.md b/docs/how-to/base-router-extensibility.md index 80fd6b6..06bdae3 100644 --- a/docs/how-to/base-router-extensibility.md +++ b/docs/how-to/base-router-extensibility.md @@ -1,148 +1,84 @@ -# 🧱 How-To: Project-Wide Base Router & Dependency Scalability +# Project-Wide Router Extensibility -## ❓ The Problem - -In a real-world ZCore project, you often need to apply **global middleware-like behavior** across all your API routes β€” for example: - -* πŸ“ Logging the request's IP address and user agent -* ⏱️ Rate-limiting or throttling -* πŸ” Enforcing tenant isolation checks -* πŸ“Š Collecting observability metrics (request duration, status codes) - -Without a centralized mechanism, you would have to duplicate these dependencies in **every single router** β€” violating the DRY principle and making maintenance a nightmare. - ---- - -## πŸ› οΈ The ZCore Solution - -ZCore's `get_route_dependencies` method makes it trivial to build a **project-wide base router** that automatically injects shared dependencies into every endpoint. This follows the same proven pattern used by Django's class-based views, Spring's `ApiController`, and .NET's controller base classes. - -### πŸ—οΈ Step 1: Create a Project-Wide Base Router - -Create a single file β€” for example `app/core/router.py` β€” that contains your application's foundational router class: - -```python -# app/core/router.py -from zcore import BaseRouter, RouteKey -from typing import Any - -class AppBaseRouter(BaseRouter[CreateSchemaType, UpdateSchemaType]): - """Foundation router for the entire application. - - Every domain router should inherit from this class to receive - project-wide dependencies automatically. - """ - - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - # 1. Inject project-wide dependencies (monitoring, logging, rate-limiting) - dependencies = [LogRequestIP(), RateLimiter()] - - # 2. Delegate to ZCore's default behavior (HasScopes, etc.) - dependencies.extend(super().get_route_dependencies(route_key, action)) - - return dependencies -``` - -### 🎯 Step 2: Domain Routers Inherit from Your Base - -Once the base router is defined, every domain router becomes dramatically simpler: - -```python -# app/products/routers.py -from app.core.router import AppBaseRouter - -class ProductRouter(AppBaseRouter): - model = Product - # That's it! Monitoring, IP logging, and security scopes are all handled automatically. -``` +Scale your API architecture by extending the BaseRouter to enforce project-wide security standards and custom endpoint behaviors. --- -## πŸ’Ž Why This Pattern Is a Scalability Masterpiece - -### 1. 🧩 DDD-Friendly: Different Bases for Different Contexts - -In large projects, you may need different behavior for different domain contexts. You can create **multiple base routers** tailored to specific needs: - -```python -# Base for admin endpoints (strict auditing, admin role check) -class AdminBaseRouter(BaseRouter[CreateSchemaType, UpdateSchemaType]): - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - dependencies = [AuditLogger(), RequireAdminRole()] - dependencies.extend(super().get_route_dependencies(route_key, action)) - return dependencies - -# Base for public endpoints (lighter monitoring) -class PublicBaseRouter(BaseRouter[CreateSchemaType, UpdateSchemaType]): - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - dependencies = [RateLimiter()] - dependencies.extend(super().get_route_dependencies(route_key, action)) - return dependencies -``` - -This level of granularity was **impossible** with the old static permission properties. - -### 2. πŸ“ Pure DRY & KISS - -* **DRY:** Global logic is written **once** in the base router file. Zero duplication across domains. -* **KISS:** Domain developers don't need to think about infrastructure concerns. They just inherit and declare their model. - -### 3. πŸš€ Superior Developer Experience (DX) - -This pattern is exactly what senior architects use in enterprise frameworks: - -| Framework | Pattern | -| :--- | :--- | -| **Django** | Class-based views with `BaseView` | -| **Spring Boot** | `@ControllerAdvice` / Base `ApiController` | -| **ASP.NET Core** | Base `ControllerBase` class | -| **ZCore** | `get_route_dependencies` in project base router | +
+
+ Type
+ Routing Architecture +
+
+ Status
+ Recommended for Scale +
+
+ Underlying Tech
+ FastAPI / Python OOP +
+
+ +## The Challenge +In a rapidly growing project, standard CRUD routers often become a source of technical debt. You might find yourself manually adding the same "Admin Only" dependency to the `DELETE` route of 20 different modules. Or perhaps you decide that every `GET_ALL` request across the entire project must include a specific rate-limiting header. + +Using raw FastAPI `APIRouter` or rigid scaffolding tools forces you to repeat this logic in every file. If the requirement changes, you have to find and replace code in dozens of places, increasing the risk of security oversights. + +## The ZCore Elegance +ZCore's `BaseRouter` is designed for **Object-Oriented Inheritance**. Instead of using it directly, you can create a `ProjectBaseRouter` for your API. By overriding the lifecycle methodsβ€”specifically `get_route_dependencies` and `get_route_action`β€”you can inject global security policies or modify the naming convention of permissions across all your domain modules from a single source of truth. + +=== "ZCore Architectural Extension" + :::python + # 1. Define your project-wide base router + class MyProjectBaseRouter(BaseRouter[CreateSchemaType, UpdateSchemaType]): + + def get_route_dependencies(self, route_key: RouteKey, action: str): + # Enforce project-wide policy: All DELETEs require 'superadmin' + deps = super().get_route_dependencies(route_key, action) + + if route_key == RouteKey.DELETE: + deps.append(HasScopes("superadmin")) + + return deps + + # 2. Use your custom base in domain routers + class UserRouter(MyProjectBaseRouter[UserCreate, UserUpdate]): + model = User + # DELETE /users now automatically requires 'users:delete' AND 'superadmin' + +=== "FastAPI Manual Repetition" + :::python + # You must remember to add the dependency to every router manually + router = APIRouter(prefix="/users") + + @router.delete("/{id}", dependencies=[Depends(HasScopes("users:delete")), Depends(HasScopes("superadmin"))]) + async def delete_user(id: uuid.UUID): + pass + + # ... Then repeat this exact dependency list in /products, /orders, /billing ... + # If the policy changes to "manager" instead of "superadmin", you edit 20 files. --- -## πŸ”§ Selective Override Within a Domain +## Boundaries & Integration +Customized routers maintain full compatibility with the FastAPI ecosystem. -Even with a project-wide base, individual domains can still override specific dependencies without losing the global behavior: - -```python -# app/orders/routers.py -from app.core.router import AppBaseRouter -from zcore.web.base_router import RouteKey - -class OrderRouter(AppBaseRouter): - model = Order - - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - # Add a domain-specific check for DELETE operations - if route_key == RouteKey.DELETE: - return [CanCancelOrder()] - - # Everything else inherits global deps + scopes via super() - return super().get_route_dependencies(route_key, action) -``` +* **Dependency Normalization:** ZCore's internal `_normalize_dependencies` helper ensures that whether you return a raw callable, a class, or a `FastAPI.Depends` object from your overrides, it is correctly wrapped and registered [web/base_router.py]. +* **Selective Overrides:** You can override a single endpoint method (e.g., `delete_endpoint`) in your project-base to implement "Soft Delete" logic project-wide, while leaving `POST` and `GET` to use the standard ZCore implementation. +* **Middleware Alternatives:** For logic that applies to *all* requests regardless of the route (like Correlation IDs), use ZCore Middlewares. Use Router Extensions for logic that varies by `RouteKey` or `Model`. --- -## πŸ§ͺ Verification - -With the `AppBaseRouter` in place, every endpoint across every domain will: - -1. πŸ“ Log the requesting IP address (`LogRequestIP`) -2. ⏱️ Apply rate-limiting (`RateLimiter`) -3. πŸ›‘οΈ Enforce the appropriate security scope (`HasScopes`) -4. 🎯 Execute the domain-specific business logic - -All of this happens **deterministically at startup** β€” zero runtime reflection, zero magic. - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The RouteKey Enum +The `RouteKey` is a `StrEnum` that maps directly to the scaffolded operations: `POST`, `GET`, `GET_ALL`, `SEARCH`, `UPDATE`, `PATCH`, and `DELETE` [web/base_router.py]. This allows your overrides to use clean, type-safe logic when determining which dependencies to inject. -!!! tip "πŸ’‘ One Method to Rule Them All" - The old architecture required maintaining up to 8 separate static class variables (`POST_PERMISSIONS`, `GET_PERMISSIONS`, etc.) across every router. The new architecture reduces this to a **single method override** β€” or zero overrides if you just need defaults. +### 2. Automatic Dependency Injection (DI) +The `BaseRouter` doesn't just register functions; it registers closures that utilize `Inject(self.service)` [web/base_router.py]. This means that even in your project-wide base router, the correct domain-specific service is always resolved from the IoC container at runtime. -!!! info "πŸ›‘οΈ Composition Over Inheritance" - While inheritance is powerful, you can also compose dependencies by creating reusable dependency factories that your base router calls. This keeps the base router itself clean and focused. +### 3. Scoped Action Derivation +The `get_route_action` method bridges the router to the model [web/base_router.py]. By default, it calls `self.model.actions().VIEW` or `CREATE`. If your project uses a different naming convention for permissions (e.g., `view_products` instead of `products:view`), you can override this single method in your `ProjectBaseRouter` to re-map actions project-wide. -!!! warning "⚠️ Order Matters" - Dependencies in the list are executed in the order they are defined. If you need authentication to run before rate-limiting, place `HasScopes` (or your auth dependency) **before** the rate limiter in the returned list. \ No newline at end of file +!!! info "Advanced Dependency Logic" + The `get_route_dependencies` method is called during the router's `__init__` phase [web/base_router.py]. This means your dependency tree is pre-compiled at startup, maintaining FastAPI's high performance during the request-response cycle. \ No newline at end of file diff --git a/docs/how-to/custom-storage-provider.md b/docs/how-to/custom-storage-provider.md index 16d1678..80399e1 100644 --- a/docs/how-to/custom-storage-provider.md +++ b/docs/how-to/custom-storage-provider.md @@ -1,187 +1,109 @@ -# πŸ“ How-To: Implementing a Custom S3 Storage Provider +# Cloud Asset Orchestration -## ❓ The Problem - -By default, ZCore includes a `LocalStorageProvider` that saves uploaded files directly to the host machine's filesystem. While this works well for development, production environments often require storing files in cloud storage solutions like AWS S3 or MinIO to support scalability and high availability. - -To switch storage backends, you need a way to swap the storage provider globally without modifying your application's route handlers or service logic. +Implement enterprise-grade cloud storage while maintaining local security defaults and rigorous file validation. --- -## πŸ› οΈ The ZCore Solution - -We suggest creating a custom storage class that inherits from ZCore's abstract `StorageProvider` base class and registering it in the IoC container during application bootstrapping. - -```mermaid -graph LR - Client[Client Upload Request] --> Router[API Router / Service] - Router -->|Resolves interface| SP[StorageProvider interface] - - subgraph IoC Container Injection - SP -->|Injected Singleton| S3[S3StorageProvider Class] - end - - S3 -->|Asynchronously upload| Target[AWS S3 Bucket / MinIO] -``` +
+
+ Type
+ Storage Protocol +
+
+ Status
+ Optional Provider +
+
+ Underlying Tech
+ anyio / aiofiles / Magic Bytes +
+
+ +## The Challenge +Handling file uploads in FastAPI is deceptively simple until production requirements hit. Developers usually face: +1. **Provider Lock-in:** Code written for `shutil.copyfileobj` (local disk) breaks when moving to AWS S3 or Google Cloud Storage. +2. **Insecure Extensions:** Relying on `file.content_type` is dangerous, as attackers can easily spoof MIME types by renaming an executable to `.jpg`. +3. **Path Traversal:** Unsanitized filenames allowing attackers to overwrite critical system files via `../../etc/passwd` exploits. +4. **Resource Exhaustion:** Large file uploads filling up server RAM or local disk space without pre-emptive size validation. + +## The ZCore Elegance +ZCore abstracts storage through the `StorageProvider` protocol. You can swap between `LocalStorageProvider` and a custom cloud provider (like S3) globally by changing a single DI registration. Every provider benefits from ZCore's multi-layered security validators, which verify file headers (Magic Bytes) and enforce strict path isolation. + +=== "ZCore S3 Provider Implementation" + :::python + from zcore.storage import StorageProvider + from fastapi import UploadFile + + class S3StorageProvider(StorageProvider): + def __init__(self, bucket: str, client: Any): + self.bucket = bucket + self.client = client + + async def upload(self, file: UploadFile, folder: str) -> str: + # Custom S3 logic + path = f"{folder}/{file.filename}" + await self.client.put_object(Bucket=self.bucket, Key=path, Body=await file.read()) + return path + + # In main.py: Global swap + container.register_singleton(StorageProvider, S3StorageProvider(bucket="my-assets", client=s3)) + +=== "FastAPI Manual Upload" + :::python + # Logic is hardcoded to one storage type + @app.post("/upload") + async def upload_file(file: UploadFile): + # 1. Manual Path Traversal Check + if ".." in file.filename: raise HTTPException(400) + + # 2. Manual Size Check + # (Requires reading part of the file or trusting headers) + + # 3. Hardcoded Storage Logic + with open(f"uploads/{file.filename}", "wb") as buffer: + shutil.copyfileobj(file.file, buffer) + + # Moving to S3 requires rewriting this entire function. --- -### πŸ“¦ Step 1: Implement the Custom S3 Storage Provider - -Create a custom storage provider that implements the abstract `upload`, `upload_stream`, and `delete` methods. - -We use the asynchronous `aioboto3` library to perform non-blocking uploads: +## Security Validation Layer +ZCore provides a suite of modular validators that run *before* the file reaches your storage target. ```python -import structlog -from typing import AsyncGenerator -from fastapi import UploadFile -import aioboto3 - -from zcore.storage.base import StorageProvider -from zcore.exceptions.base import AppException - -logger = structlog.get_logger() - -class S3StorageProvider(StorageProvider): - """Custom cloud storage integration mapping ZCore operations to AWS S3 / MinIO.""" - - def __init__( - self, - aws_access_key_id: str, - aws_secret_access_key: str, - bucket_name: str, - region_name: str = "us-east-1", - endpoint_url: str | None = None # Use for custom targets like MinIO - ) -> None: - self.session = aioboto3.Session( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - region_name=region_name - ) - self.bucket_name = bucket_name - self.endpoint_url = endpoint_url - - async def upload(self, file: UploadFile, folder: str) -> str: - """Upload an API file to the configured S3 bucket.""" - target_path = f"{folder}/{file.filename}" - - try: - async with self.session.client("s3", endpoint_url=self.endpoint_url) as s3: - # Read file payload - file_data = await file.read() - - await s3.put_object( - Bucket=self.bucket_name, - Key=target_path, - Body=file_data, - ContentType=file.content_type - ) - logger.info(f"Successfully uploaded file to S3: s3://{self.bucket_name}/{target_path}") - return target_path - except Exception as e: - logger.error(f"Failed to upload file to S3: {e}") - raise AppException("Error uploading file to cloud storage.") - - async def upload_stream( - self, - file_stream: AsyncGenerator[bytes, None], - filename: str, - folder: str - ) -> str: - """Stream file bytes directly to S3.""" - target_path = f"{folder}/{filename}" - - try: - async with self.session.client("s3", endpoint_url=self.endpoint_url) as s3: - # To support multipart streaming in S3 - mpu = await s3.create_multipart_upload(Bucket=self.bucket_name, Key=target_path) - parts = [] - part_number = 1 - - async for chunk in file_stream: - part = await s3.upload_part( - Bucket=self.bucket_name, - Key=target_path, - PartNumber=part_number, - UploadId=mpu["UploadId"], - Body=chunk - ) - parts.append({"PartNumber": part_number, "ETag": part["ETag"]}) - part_number += 1 - - await s3.complete_multipart_upload( - Bucket=self.bucket_name, - Key=target_path, - UploadId=mpu["UploadId"], - MultipartUpload={"Parts": parts} - ) - return target_path - except Exception as e: - logger.error(f"S3 streaming upload failed: {e}") - raise AppException("Error streaming file payload to cloud storage.") - - async def delete(self, file_path: str) -> bool: - """Remove a file from the S3 bucket.""" - try: - async with self.session.client("s3", endpoint_url=self.endpoint_url) as s3: - await s3.delete_object(Bucket=self.bucket_name, Key=file_path) - return True - except Exception as e: - logger.error(f"Failed to delete S3 asset: {e}") - return False +from zcore.storage import MaxFileSizeValidator, SafeMimeTypeValidator + +# Configure local storage with strict security +storage = LocalStorageProvider( + base_path="./uploads", + validators=[ + MaxFileSizeValidator(max_size_mb=5), + SafeMimeTypeValidator(allowed_mimes=["image/png", "image/jpeg", "application/pdf"]) + ] +) ``` --- -### βš™οΈ Step 2: Register the Provider in the IoC Container +## Boundaries & Integration +The storage layer is a decoupled utility that respects FastAPI's standard types. -To configure your application to use the S3 provider, register it as a global singleton in the IoC container during application bootstrapping: - -```python -# main.py -from zcore.kernel.di import container -from zcore.storage.base import StorageProvider -from zcore.config import settings -from .storage import S3StorageProvider - -# Instantiate the custom S3 provider -s3_provider = S3StorageProvider( - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - bucket_name=settings.AWS_BUCKET_NAME, - endpoint_url=settings.AWS_S3_ENDPOINT_URL # Optional: configure for MinIO -) - -# Register the provider as a singleton in the IoC container -container.register_singleton(StorageProvider, s3_provider) -``` +* **FastAPI `UploadFile`:** The `upload` method accepts the standard Starlette/FastAPI `UploadFile` object, ensuring compatibility with native multipart form handling. +* **Dependency Injection:** Use `get_storage_provider` as a FastAPI dependency to resolve the active provider in any route, regardless of whether it is local or cloud-based. +* **AnyIO Integration:** The `LocalStorageProvider` uses `anyio.Path` and `aiofiles` for non-blocking I/O, ensuring that file operations do not freeze the main event loop. --- -### 🚦 Step 3: Usage in Your Application +## Under-the-Hood Spec -Because your services resolve the storage provider through the IoC container, you can use S3 storage throughout your application without changing your business logic: +### 1. Magic Byte Content Verification +The `SafeMimeTypeValidator` does not trust the file extension or the browser-supplied `content_type` [storage/validators.py]. It reads the first **2048 bytes** of the file and compares them against hardcoded "Magic Byte" signatures (e.g., `\x89PNG\r\n\x1a\n`). It also performs a specific security check to block high-risk patterns like `|Extract Role| Check{Is User Admin?} - Check -->|No: Standard User| Bind[set_restricted_fields: employee.salary & employee.bank_account_number] - Check -->|Yes: Admin| Exec[Execute business logic query] - - Bind --> Exec - Exec -->|Returns Zchema model| Serialize[Zchema.secure_serializer] - Serialize -->|Prune restricted fields| Output[JSON Response payload without sensitive data] -``` +
+
+ Type
+ Context Shielding +
+
+ Status
+ Recommended Core +
+
+ Underlying Tech
+ Pydantic V2 / contextvars +
+
+ +## The Challenge +Modern APIs often deal with "Multifaceted Data"β€”where the same record (e.g., a **User Profile**) has different visible fields depending on who is asking. +1. **The Public** should only see `username` and `avatar`. +2. **The Owner** should also see `email` and `billing_address`. +3. **The Admin** should see everything, including `internal_notes` and `risk_score`. + +In standard FastAPI, this usually forces developers to create a separate Pydantic model for every role (`UserPublic`, `UserPrivate`, `UserAdmin`) and write complex `if/else` logic in routers to choose the correct model. This is error-prone and causes "Schema Explosion." + +## The ZCore Elegance +ZCore's `Zchema` allows you to define **one model** for all roles. By setting the `__model__` attribute, the schema becomes "context-aware." When the request starts, your security middleware populates a list of `restricted_fields` (e.g., `["users.internal_notes"]`). ZCore then automatically strips these fields from incoming JSON payloads (Mass Assignment protection) and outgoing JSON responses (Leakage protection). + +=== "ZCore Context Masking" + :::python + from zcore import Zchema, request_context + + class UserSchema(Zchema): + __model__ = "users" # Ties to the 'users' domain + + username: str + email: str + internal_notes: str | None = None + + # Inside a middleware or dependency: + restricted = ["users.internal_notes"] + if not user.is_admin: + with request_context(user_id=user.id, fields=restricted): + # Outgoing response now automatically + # masks 'internal_notes' + return UserSchema.model_validate(user_record) + +=== "FastAPI Manual Projection" + ::python + # Multiple models required + class UserPublic(BaseModel): + username: str + + class UserPrivate(UserPublic): + email: str + internal_notes: str + + @router.get("/users/{id}") + async def get_user(id: uuid.UUID, user: User = Depends(get_user)): + record = await db.get(id) + # Manual role-based branching logic + if user.is_admin: + return UserPrivate.model_validate(record) + return UserPublic.model_validate(record) --- -### πŸ“¦ Step 1: Define Your Schema with `Zchema` - -All schemas must inherit from `Zchema` and declare a `__db_name__` that binds them to their database domain: - -```python -from zcore import Zchema -from pydantic import Field, ConfigDict -import uuid +## Boundaries & Integration +ZCore's masking logic is a non-destructive layer built on top of Pydantic. -class EmployeeResponse(Zchema): - __db_name__ = "employee" - id: uuid.UUID - name: str - salary: float # Will be pruned for non-admin users - bank_account_number: str # Will be pruned for non-admin users - - model_config = ConfigDict(from_attributes=True) -``` +* **Pydantic Native:** Pruning happens during Pydantic's internal `model_validator` and `model_serializer` phases [web/projection.py]. Your logic inside the service or repository layers remains untouchedβ€”you always work with full objects. +* **Opt-In Masking:** If a schema does not define `__model__`, ZCore ignores it. It behaves like a standard `BaseModel`. +* **Freedom to Bypass:** If you need to force-send a restricted field for a specific edge case, you can use a standard `pydantic.BaseModel` for that specific route, which will bypass ZCore's context-aware pruning. --- -### πŸ›‘οΈ Step 2: Implement a Scoped Restriction Dependency - -Create a FastAPI dependency that evaluates the authenticated user's permissions and binds restricted fields to the context using the `{db_name}.{field}` syntax: - -```python -from fastapi import Depends -from zcore.context import set_restricted_fields -from zcore.security.dependencies import get_current_user_stub -from zcore.security.protocols import UserProtocol - -async def apply_security_field_restrictions( - user: UserProtocol = Depends(get_current_user_stub) -) -> None: - """Evaluate user roles and bind restricted fields to the active context.""" - restricted_paths = set() - - # If the user is not a superuser, restrict sensitive fields - if not getattr(user, "is_superuser", False): - # Use the domain-specific namespace: {db_name}.{field} - restricted_paths.update([ - "employee.salary", - "employee.bank_routing_number", - "employee.bank_account_number" - ]) - - # Bind the restricted paths to the request-scoped context - set_restricted_fields(restricted_paths) -``` - ---- - -### πŸ—„οΈ Step 3: Set Up Your Database Model & Router - -Define your database model and route configurations as usual. ZCore's router automatically integrates with the `Zchema` security pipeline: - -```python -# models.py -import uuid -from sqlalchemy.orm import Mapped, mapped_column -from sqlalchemy import String, Numeric -from zcore.db.setup import Base - -class Employee(Base): - __tablename__ = "employees" - - id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) - name: Mapped[str] = mapped_column(String(100)) - salary: Mapped[float] = mapped_column(Numeric(10, 2), nullable=False, default=0.0) - stock: Mapped[int] = mapped_column(Integer, nullable=False, default=0) -``` - -For the web layer, inject the security restrictions via the `get_route_dependencies` method: - -```python -# routers.py -from typing import Any -from zcore.web.base_router import BaseRouter, RouteKey -from zcore.db.pagination import PageNumberPagination - -from .schemas import EmployeeCreate, EmployeeUpdate, EmployeeResponse -from .services import EmployeeService -from .models import Employee -from .security import apply_security_field_restrictions - -class EmployeeRouter(BaseRouter[EmployeeCreate, EmployeeUpdate]): - model = Employee - create_schema = EmployeeCreate - update_schema = EmployeeUpdate - schema_out = EmployeeResponse - service = EmployeeService - - prefix = "/employees" - tags = ["Employees"] - - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - """Inject field restriction dependencies for view operations.""" - if route_key in (RouteKey.GET, RouteKey.GET_ALL): - return [Depends(apply_security_field_restrictions)] - return super().get_route_dependencies(route_key, action) -``` - ---- - -### πŸ§ͺ Step 4: Verification - -When a non-admin user requests employee data (e.g. `GET /employees/{id}`), ZCore intercepts the response during serialization via the `Zchema.secure_serializer` hook. - -Even if your service layer loads the full `Employee` database object with its salary and bank details, the outgoing JSON payload is dynamically pruned: - -```json -{ - "success": true, - "message": "Success", - "data": { - "id": "b1b0e363-26a1-438f-9aef-88df5ee47fa2", - "name": "Alex Mercer" - // salary and bank_account_number are automatically pruned! - } -} -``` - ---- +## Under-the-Hood Spec -## πŸ’‘ Engineering Insights +### 1. The Recursive `_prune_data` Logic +ZCore doesn't just check top-level keys. The internal `_prune_data` method recursively traverses nested dictionaries and lists [web/projection.py]. If you restrict `orders.items.cost`, ZCore will find every `cost` field inside the `items` list of an `Order` object and remove it, regardless of the nesting depth. -!!! tip "πŸ’‘ Why `{db_name}.{field}` Instead of `resource.{field}`?" - The old `resource.` prefix caused namespace collisions across different domains. The new `{db_name}.{field}` syntax (e.g., `employee.salary`, `user.email`) ensures strict domain boundary isolation, preventing fields in the `billing` module from accidentally affecting fields in the `crm` module. +### 2. Dynamic JSON-Schema Pruning +When `expose_schemas` is enabled on a router, ZCore overrides `__get_pydantic_json_schema__` [web/projection.py]. It doesn't just mask the data; it **masks the documentation**. If a user lacks permission to see `internal_notes`, that field will not appear in the JSON-Schema returned by `?schema=true`, allowing your frontend to dynamically hide form inputs. -!!! info "πŸ›‘οΈ Three Tiers of Protection" - `Zchema` doesn't just prune responses. It also: - 1. **Prunes JSON Schemas** when clients request `?schema=true` - 2. **Strips restricted fields from incoming payloads** (prevents Mass Assignment) - 3. **Prunes outgoing responses** during native Pydantic serialization +### 3. Cache-Safe `Vary` Headers +Because the same URL can return different JSON content based on the user's permissions, ZCore's `ZCoreAPIRoute` automatically appends `Authorization` or `Cookie` to the `Vary` HTTP header [web/api_router.py]. This ensures that downstream CDNs or browser caches do not accidentally serve a cached "Admin" response to a "Public" user. -!!! note "🧠 Caching Safety" - ZCore's `ZCoreAPIRoute` automatically appends `Authorization` or `Cookie` parameters to the response's `Vary` header whenever restricted fields are active. This instructs upstream reverse-proxies that the response varies based on user credentials, preventing data leaks across shared caches. \ No newline at end of file +!!! info "Security Policy" + Restricted fields are defined as dot-paths (e.g., `model_name.field_name`). To restrict an entire model, you can use the model name itself (e.g., `users`). This will cause the entire schema to return an empty object if the user is unauthorized. \ No newline at end of file diff --git a/docs/how-to/multi-repo-transactions.md b/docs/how-to/multi-repo-transactions.md index a7500cb..1746f74 100644 --- a/docs/how-to/multi-repo-transactions.md +++ b/docs/how-to/multi-repo-transactions.md @@ -1,113 +1,99 @@ -# πŸ”— How-To: Atomic Transactions Across Multiple Repositories +# Multi-Repository Transactions -## ❓ The Problem +Ensure data integrity across multiple domain services with atomic Unit of Work orchestration and post-commit event dispatching. -In complex business scenarios, an API request often needs to update multiple database tables at once. For example, when checking out, you must insert an `Order` and deduct items from the `Product` inventory. +--- -If you commit each database modification separately, any failure halfway through the process (like running out of stock after the order is saved) will leave your database in an inconsistent state. To prevent this, you need to manage these changes atomically within a single transactional boundary. +
+
+ Type
+ Transaction Orchestrator +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / Unit of Work +
+
+ +## The Challenge +When business logic spans multiple domains (e.g., creating an **Order**, updating **Inventory**, and deducting **User Balance**), atomicity is critical. If the database update for the balance fails, the order and inventory must not persist. + +In raw FastAPI/SQLAlchemy code, developers often struggle with: +1. **Premature Commits:** A sub-service calls `db.commit()` inside its own method, finalizing part of the transaction before the rest is ready. +2. **Ghost Events:** A domain event (like an "Order Confirmation Email") is sent to a message broker, but the database transaction subsequently fails and rolls back. The user receives an email for an order that doesn't exist. +3. **Session Pollution:** Passing a single session through five different function signatures, making the code brittle and hard to test. + +## The ZCore Elegance +ZCore utilizes the **Unit of Work (UOW)** pattern combined with a metadata-aware session [uow.py]. When you wrap operations in a `UnitOfWork` context, it marks the session as `uow_managed`. ZCore's `BaseService` detects this flag and automatically skips its internal commits, delegating the final authority to the UOW. Furthermore, events registered during the transaction are buffered and only dispatched if the commit succeeds. + +=== "ZCore Unit of Work" + :::python + from zcore import UnitOfWork, Inject + from zcore.kernel.events import EventDispatcher + + async def checkout_service( + order_data: OrderCreate, + order_service: OrderService = Inject(OrderService), + inventory_service: InventoryService = Inject(InventoryService), + session: AsyncSession = SessionDep, + dispatcher: EventDispatcher = Inject(EventDispatcher) + ): + # Atomic Boundary + async with UnitOfWork(session, dispatcher) as uow: + # Sub-services detect UOW and skip internal commits + order = await order_service.create(order_data) + await inventory_service.reduce_stock(order.items) + + # Queue event for post-commit dispatch + uow.register_event("order.placed", {"id": order.id}) + + # Commit happens automatically at the end of 'async with' + +=== "FastAPI Manual Transaction" + :::python + @app.post("/checkout") + async def checkout(data: OrderCreate, db: AsyncSession = Depends(get_db)): + try: + # 1. Manual order logic + order = await create_order(db, data) + # 2. Manual inventory logic + await reduce_stock(db, order.items) + + # 3. Manual commit + await db.commit() + + # 4. Manual event trigger + # If this fails, the user is confused, but the DB is committed. + await dispatcher.dispatch("order.placed", {"id": order.id}) + except Exception: + await db.rollback() + raise --- -## πŸ› οΈ The ZCore Solution - -We suggest using ZCore's `UnitOfWork` pattern to wrap multiple repositories in a single transactional block. This ensures that all database flushes are rolled back if any operation fails, and buffers any domain events until the transaction successfully commits. - -```mermaid -sequenceDiagram - participant App as CheckoutService - participant UOW as UnitOfWork - participant DB as SQLite / Postgres Session - participant ED as EventDispatcher - - Note over App,UOW: Enter transactional context manager block - UOW->>DB: Set session.info['uow_managed'] = True - - App->>DB: 1. Create Order (via OrderRepository.create) - DB-->>App: Order record flushed - - App->>DB: 2. Deduct Inventory (via ProductRepository.update) - Note over App: Price is negative or stock is insufficient! - App-->>UOW: Raise ValidationError - - Note over UOW: Exception raised! Trigger clean rollback - UOW->>DB: rollback() - Note over UOW: Clear pending events queue - UOW-->>App: Propagate ValidationError to client -``` +## Boundaries & Integration +ZCore provides a structured wrapper around SQLAlchemy's native transaction lifecycle. + +* **Session Native:** `UnitOfWork` operates on a standard `AsyncSession`. You can still call `session.execute()` directly inside a UOW block. +* **Event Decoupling:** The `EventDispatcher` is a core ZCore component. The UOW strictly manages the *timing* of the dispatch, not the handler logic itself. +* **Freedom to Bypass:** If you do not use the `UnitOfWork` context, `BaseService` defaults to its `_safe_commit` behavior, which commits the transaction immediately after the service operation concludes. --- -### πŸ’» Implementation Example: The Checkout Flow - -In the following example, the service uses `ProductRepository` and `OrderRepository` inside a `UnitOfWork` block to process checkouts atomically: - -```python -import uuid -from zcore.service.base import BaseService -from zcore.db.uow import UnitOfWork -from zcore.kernel.di import Inject -from zcore.exceptions.base import ValidationError - -from .models import Order -from .schemas import OrderCreate, OrderUpdate -from .repositories import OrderRepository, ProductRepository - -class CheckoutService(BaseService[Order, OrderCreate, OrderUpdate]): - - def __init__( - self, - repository: OrderRepository = Inject(OrderRepository), - product_repo: ProductRepository = Inject(ProductRepository) - ) -> None: - super().__init__(model=Order, repository=repository) - self.product_repo = product_repo - - async def process_checkout( - self, - user_id: uuid.UUID, - product_id: uuid.UUID, - quantity: int - ) -> Order: - """Process an order checkout atomically.""" - - # 1. Access the request-scoped database session and the event dispatcher - session = self.repository.db - dispatcher = self.repository.db.info.get("event_dispatcher") # Fetched from kernel context - - # 2. Wrap operations in the UnitOfWork context manager - async with UnitOfWork(session, dispatcher) as uow: - - # Fetch and check product inventory - product = await self.product_repo.get(product_id) - if not product or product.stock < quantity: - raise ValidationError(message="Cannot complete checkout: insufficient product stock.") - - # Deduct items from product inventory (Repository changes are flushed, not committed) - product.stock -= quantity - await self.product_repo.db.flush() - - # Create the order record - order_schema = OrderCreate( - user_id=user_id, - product_id=product_id, - quantity=quantity, - total_price=product.price * quantity - ) - order = await self.repository.create(order_schema) - - # Register post-commit domain events with the Unit of Work - uow.register_event("order.completed", {"order_id": str(order.id), "user_id": str(user_id)}) - - # The context manager automatically commits the transaction here if no errors occurred - return order -``` +## Under-the-Hood Spec ---- +### 1. The `uow_managed` Flag +Inside `UnitOfWork.__aenter__`, the framework sets `self.session.info["uow_managed"] = True` [uow.py]. The `BaseService._safe_commit` method checks this specific key in the session's internal dictionary [service/base.py]. If present, it treats the session as "borrowed" and skips the `commit()` call, ensuring that only the top-level orchestrator decides when to finalize the data. -## πŸ“ Technical Insights +### 2. Transactional Event Buffering +The `UnitOfWork` maintains an internal `_pending_events` list [uow.py]. When `uow.register_event()` is called, the event is simply stored in memory. The dispatcher is only invoked in the `commit()` method *after* `await self.session.commit()` has successfully returned. If the database commit raises an exception, the event list is cleared during `rollback()`, preventing "ghost notifications." -!!! note "πŸ›‘οΈ The Role of 'uow_managed'" - During repository and service calls (like `self.repository.create`), ZCore checks if `self.repository.db.info["uow_managed"]` is set to `True`. Because the transaction is wrapped in a `UnitOfWork` block, individual repository commits are bypassed. The database session is only committed when the context manager block exits successfully. +### 3. Automatic Exception Handling +The `__aexit__` method of the `UnitOfWork` is designed for safety [uow.py]. If an exception is raised within the `async with` block, it automatically triggers `await self.rollback()`. If the block completes successfully, it triggers `await self.commit()`. This eliminates the need for manual `try/except/rollback` blocks in your business orchestrators. -!!! warning "⚠️ Handling Event Isolation" - If the transaction rolls back, all registered events are discarded. If the transaction commits successfully, the events are dispatched. If any event listener fails during dispatching, the error is caught and logged, ensuring that listener failures do not affect the already committed database transaction. \ No newline at end of file +!!! info "Concurrency Note" + Events dispatched by the Unit of Work use the `EventDispatcher.dispatch` method, which executes all registered asynchronous handlers concurrently using `asyncio.gather`. This ensures that post-transaction side effects do not serialize your application's response time. \ No newline at end of file diff --git a/docs/how-to/websocket-pubsub.md b/docs/how-to/websocket-pubsub.md index 8cdf17c..28f567e 100644 --- a/docs/how-to/websocket-pubsub.md +++ b/docs/how-to/websocket-pubsub.md @@ -1,515 +1,97 @@ -# πŸ“ ZCore Framework Quick Reference (Cheat Sheet) - -This cheat sheet serves as a quick-access reference guide for standard syntax, CLI commands, and configurations across the ZCore framework's 27 architectural layers and capabilities. - ---- - -## πŸ–₯️ Block 1: Bootstrapping & CLI - -### πŸ’» 1. CLI Reference (`cli`) -The command-line utility for project scaffolding, application setup, and local development execution. - -| Command | Example Usage | Description | -| :--- | :--- | :--- | -| `init` | `zc init billing_api` | Initializes a new ZCore project workspace. | -| `startapp` | `zc startapp checkout -t` | Scaffolds a new domain application with boilerplate code. | -| `run` | `zc run` | Launches the local reload-enabled development server. | -| `gensecret` | `zc gensecret` | Generates a secure, 64-character secret key. | - -πŸ‘‰ [CLI Reference Documentation](features/infrastructure/cli.md) - ---- - -### βš™οΈ 2. Configuration (`config`) -Loads environmental configuration maps securely with fallback settings and lazy resolution through the settings proxy. - -```python -from zcore.config import settings - -# Lazy attribute resolution avoids premature instantiation during import time -db_url = settings.DATABASE_URL -token_lifespan = settings.ACCESS_TOKEN_EXPIRE_MINUTES -``` - -πŸ‘‰ [Configuration Reference Documentation](features/infrastructure/config.md) - ---- - -### πŸš€ 3. Application Entry (`main.py`) -Bootstrapping boilerplate that brings together logging, dbs, routers, plugins, and the core lifecycle kernel. - -```python -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from fastapi import FastAPI -from zcore import Kernel, settings -from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware -from zcore.exceptions import app_exception_handler, AppException -from zcore.db import db_manager, register_db_event_dispatcher -from zcore.logging import setup_logging - -setup_logging() -db_manager.init_app(db_url=settings.DATABASE_URL) - -kernel = Kernel() -register_db_event_dispatcher(kernel.dispatcher) - -app = FastAPI(title=settings.PROJECT_NAME, lifespan=kernel.lifespan) -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) -app.add_exception_handler(AppException, app_exception_handler) - -kernel.setup(app) -``` - -πŸ‘‰ [Application Entry Reference Documentation](features/infrastructure/main.md) - ---- - -## 🧱 Block 2: Modular App Architecture - -### πŸ—„οΈ 4. DB Infrastructure (`db`) -Coordinates the database connection engine and registers table metadata for permission checking. - -```python -from zcore.db import db_manager - -# Initialize pool tuning and SQL query logging options -db_manager.init_app(db_url="sqlite+aiosqlite:///dev.db", pool_size=5, max_overflow=10, echo=True) -``` - -πŸ‘‰ [Database Infrastructure Reference Documentation](features/database/db.md) - ---- - -### πŸ“Š 5. Database Model (`db models`) -SQLAlchemy 2.0 declarative database model mapping, using UUIDv4 as the default identifier. - -```python -import uuid -from sqlalchemy import String, Numeric -from sqlalchemy.orm import Mapped, mapped_column -from zcore.db.setup import Base - -class Product(Base): - __tablename__ = "products" - - id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) - name: Mapped[str] = mapped_column(String(100), nullable=False) - price: Mapped[float] = mapped_column(Numeric(10, 2), nullable=False) -``` - -πŸ‘‰ [Database Model Reference Documentation](learn/1-db.md) - ---- - -### πŸ“‹ 6. Pydantic Schemas (`schemas`) -Pydantic V2 data payload definitions that enforce validation constraints and map database outputs. All schemas should inherit from `Zchema` and declare a `__db_name__` for domain-bound security. - -```python -from zcore import Zchema -from pydantic import Field, ConfigDict - -class ProductResponse(Zchema): - __db_name__ = "product" - id: str - name: str - - # Enables automatic mapping from database models - model_config = ConfigDict(from_attributes=True) -``` - -πŸ‘‰ [Schemas Reference Documentation](learn/2-schemas.md) - ---- - -### πŸ›οΈ 7. Repository Layer (`repository`) -Data access layer that provides standard CRUD query operations directly with zero SQL boilerplate. - -```python -from zcore.db.repository import BaseRepository -from zcore.db.setup import SessionDep -from .models import Product - -class ProductRepository(BaseRepository[Product, ProductCreate, ProductUpdate]): - def __init__(self, db: SessionDep): - super().__init__(model=Product, db=db) -``` - -πŸ‘‰ [Repository Layer Reference Documentation](features/database/repository.md) - ---- - -### βš™οΈ 8. Business Service (`service`) -The business logic layer, featuring integrated dependency injection, `on_*` persistence methods, and pre/post lifecycle hooks. - -```python -from zcore.service.base import BaseService -from zcore.kernel.di import Inject -from .models import Product -from .schemas import ProductCreate, ProductUpdate -from .repositories import ProductRepository - -class ProductService(BaseService[Product, ProductCreate, ProductUpdate]): - def __init__(self, repository: ProductRepository = Inject(ProductRepository)): - super().__init__(model=Product, repository=repository) -``` - -πŸ‘‰ [Business Service Reference Documentation](features/database/service.md) - ---- - -### 🌐 9. Routing Scaffolding (`BaseRouter`) -A generic router class that automatically generates and secures standard REST endpoints. - -```python -from zcore.web import BaseRouter -from .schemas import ProductCreate, ProductUpdate, ProductResponse -from .services import ProductService -from .models import Product - -class ProductRouter(BaseRouter[ProductCreate, ProductUpdate]): - model = Product - create_schema = ProductCreate - update_schema = ProductUpdate - schema_out = ProductResponse - service = ProductService - prefix = "/products" -``` - -πŸ‘‰ [Routing Scaffolding Reference Documentation](learn/5-router.md) - ---- - -### πŸ—οΈ 10. Plugin System (`Plugin`) -Packages modular components to handle initialization logic and coordinate application lifecycles. - -```python -from fastapi import FastAPI -from zcore.kernel import Plugin - -class ProductPlugin(Plugin): - name = "products" - version = "1.0.0" - dependencies = [] - - def setup(self, app: FastAPI) -> None: - app.include_router(product_router) - - async def on_startup(self) -> None: - pass # Handle startup tasks (e.g. warming caches) -``` - -πŸ‘‰ [Plugin System Reference Documentation](features/infrastructure/kernel.md) - ---- - -### 🧠 11. Framework Core (`engine - kernel`) -The central engine that registers, topologically sorts, and triggers application modules during server startup. - -```python -from zcore import Kernel -from products.plugin import ProductPlugin - -kernel = Kernel() -# Register the plugin to include its modules during the startup phase -kernel.add_plugin(ProductPlugin()) -``` - -πŸ‘‰ [Framework Core Reference Documentation](features/infrastructure/kernel.md) - ---- - -## πŸ”Œ Block 3: DI, Middlewares & Utilities - -### πŸ’‰ 12. Dependency Injection (`di`) -Registers and resolves classes. It supports Scoped, Transient, and Singleton lifecycles. - -```python -from zcore.kernel.di import container, Inject - -# Register singleton -container.register_singleton(IMailer, SendGridMailer()) - -# Register scoped class (request lifetime) -container.register_scoped(IOrderRepository, SQLOrderRepository) - -# Inject dependencies directly into constructors -def __init__(self, repo: IOrderRepository = Inject(IOrderRepository)): - self.repo = repo -``` - -πŸ‘‰ [Dependency Injection Reference Documentation](features/infrastructure/di.md) - ---- - -### πŸ”” 13. Kernel Events (`kernel - event`) -Subscribes to and dispatches events asynchronously, running listeners concurrently on the active event loop. - -```python -from zcore.kernel.events import EventDispatcher - -# Subscribe a listener to an event key -dispatcher.subscribe("order.created", send_invoice_email) - -# Dispatch an event concurrently across all registered listeners -await dispatcher.dispatch("order.created", order_id="ord_102") -``` - -πŸ‘‰ [Kernel Events Reference Documentation](features/infrastructure/events.md) - ---- - -### πŸ›‘οΈ 14. ASGI Middlewares (`middleware`) -Configures tracing and sets up request-scoped memory environments to isolate concurrent requests. - -```python -# Registered in main.py to handle request tracking and dependency isolation -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) -``` - -πŸ‘‰ [ASGI Middlewares Reference Documentation](features/infrastructure/middleware.md) - ---- - -### πŸ› οΈ 15. Utilities (`utils`) -Serializes non-primitive types and validates structures using JSON standard validation rules. - -```python -from zcore.utils.helpers import json_dumps -from zcore.utils.jsonschema import validate_json_schema - -# Serialize UUIDs, Decimals, and Dates cleanly -serialized_payload = json_dumps({"id": uuid.uuid4(), "amount": Decimal("10.50")}) - -# Validate data structures against a Draft-7 schema -validate_json_schema(data={"age": 20}, schema={"properties": {"age": {"type": "integer"}}}) -``` - -πŸ‘‰ [Utilities Reference Documentation](features/infrastructure/utils.md) - ---- - -## πŸ“Š Block 4: Transactions, Search & Pagination - -### πŸ”— 16. Multi-Repo Transactions (`uow`) -Executes transactional writes across multiple repositories in a single block, ensuring atomic changes. - -```python -from zcore.db.uow import UnitOfWork - -# Wrap multiple changes in a single database transaction -async with UnitOfWork(session, dispatcher) as uow: - await order_repo.create(order_schema) - await product_repo.update(product_id, update_schema) - uow.register_event("checkout.completed", {"order_id": 102}) -``` - -πŸ‘‰ [Multi-Repo Transactions Reference Documentation](features/database/uow.md) - ---- - -### πŸ”” 17. Database Events (`db events`) -Signals database updates and dispatches them asynchronously, keeping transaction execution non-blocking. - -```python -from zcore.db.events import dispatch_db_event - -# Safely publish database events across the system -await dispatch_db_event("db.product.created", {"id": "prod_102"}) -``` - -πŸ‘‰ [Database Events Reference Documentation](features/database/db-events.md) - ---- - -### πŸ“Š 18. DB Pagination (`pagination`) -ZCore provides Page-Number and Keyset Cursor strategies to handle dataset chunk loading. - -```python -from zcore.db.pagination import PageNumberParams, CursorParams - -# Offset parameters -offset_params = PageNumberParams(page=1, size=20, sort_by="price", sort_order="desc") - -# Keyset parameters (using base64 encoded cursor positions) -cursor_params = CursorParams(cursor="eyJ2YWx1ZSI6MjAsImlkIjoyfQ==", size=25) -``` - -πŸ‘‰ [DB Pagination Reference Documentation](features/database/pagination.md) - ---- - -### πŸ” 19. Dynamic Search (`search`) -Builds complex database queries dynamically, validating paths and converting types securely. - -```json -{ - "filters": [ - { "field": "stock", "op": "gt", "value": 10 }, - { "op": "or", "items": [ - { "field": "name", "op": "ilike", "value": "coffee" }, - { "field": "tags.name", "op": "eq", "value": "organic" } - ] - } - ], - "include": ["category"], - "sort": [{ "field": "price", "order": "asc" }] -} -``` - -πŸ‘‰ [Dynamic Search Reference Documentation](features/database/search.md) - ---- - -## πŸ›‘οΈ Block 5: Security & Exceptions - -### 🧠 20. Context Management (`context`) -Stores user and field restriction parameters securely across async execution blocks. Restricted fields use the `{db_name}.{field}` syntax for domain-bound security. - -```python -from zcore.context import request_context, get_current_user_id - -# Isolate user context and field restrictions for a specific block -with request_context(user_id=active_user_id, fields=["user.hashed_password"]): - # Retrieve the active user ID within any nested function block - current_user = get_current_user_id() -``` - -πŸ‘‰ [Context Management Reference Documentation](features/security/context.md) - ---- - -### πŸ”’ 21. Cryptographic Security (`security`) -Hashes passwords using the Argon2id algorithm and issues cryptographically signed JWT credentials. - -```python -from zcore.security import get_password_hash, create_token - -# Generate a secure Argon2id password hash -hashed_password = get_password_hash("raw-password-string") - -# Issue a new signed JWT access token containing claims -token = create_token(data={"sub": "user_id_102", "role": "admin"}) -``` - -πŸ‘‰ [Cryptographic Security Reference Documentation](features/security/security.md) - ---- - -### 🚦 22. Custom Route Mechanics (`api-router`) -Enables custom routing rules, including dynamic schema exposure and native Zchema-based security. - -```python -# Enables /route/?schema=true queries on the endpoint -expose_schemas = True -``` - -πŸ‘‰ [Custom Route Mechanics Reference Documentation](features/security/api-router.md) - ---- - -### βœ‚οΈ 23. Output Projection & Security (`Zchema & response`) -ZCore's `Zchema` base class automatically prunes restricted fields across schema generation, input validation, and response serialization β€” all without web-layer middleware. - -```python -from zcore import Zchema -from zcore.web.response import ResponseWrapper - -class EmployeeResponse(Zchema): - __db_name__ = "employee" - id: str - name: str - salary: float # Automatically pruned for non-admin users - -# Package responses in the standardized JSON envelope -response = ResponseWrapper.success_response(data=employee_data, message="Success") -``` - -πŸ‘‰ [Zchema Security Reference Documentation](features/security/zchema.md) - ---- - -### πŸ›‘ 24. Global Exceptions (`exceptions`) -Standardizes custom application exception mappings and converts errors into clean JSON error responses. - -```python -from zcore.exceptions.base import ValidationError, EntityNotFound - -# Raise to return an HTTP 400 Bad Request response containing metadata -raise ValidationError(message="Validation failed", payload={"field": "email"}) - -# Raise to return an HTTP 404 Not Found response -raise EntityNotFound(message="Resource not found") -``` - -πŸ‘‰ [Global Exceptions Reference Documentation](features/security/exceptions.md) - ---- - -## 🌐 Block 6: Advanced Ecosystem Services - -### ⚑ 25. Hybrid Caching (`cache`) -Caches data using local in-memory storage, falling back cleanly to Redis connections if available. - -```python -from zcore.cache import BaseCache - -cache = BaseCache[UserSchema](prefix="users") - -# Set cache key -await cache.set(key="user_102", value=user_data, ttl=3600) - -# Retrieve cached payload using Pydantic schema validation -cached_user = await cache.get("user_102", target_type=UserSchema) -``` - -πŸ‘‰ [Hybrid Caching Reference Documentation](features/ecosystem/cache.md) - ---- - -### πŸ“ 26. File Storage (`storage`) -Manages file uploads securely, protecting against path traversal and validating files based on their Magic Bytes. - -```python -from zcore.storage.local import LocalStorageProvider -from zcore.storage.validators import SafeMimeTypeValidator - -# Initialize local storage provider with image magic-byte security validators -storage = LocalStorageProvider( - base_path="./uploads", - validators=[SafeMimeTypeValidator(allowed_mimes=["image/png", "image/jpeg"])] -) - -# Upload and save file safely -saved_path = await storage.upload(file, folder="profile_pictures") -``` - -πŸ‘‰ [File Storage Reference Documentation](features/ecosystem/storage.md) - ---- - -### πŸ“‘ 27. Event Streaming (`stream`) -Streams real-time updates over WebSocket channels, using Redis PubSub to route events across instances. - -```python -from zcore.web.streams import StreamManager - -stream_manager = StreamManager() - -# Stream real-time events to a connected client -async with stream_manager.subscription(user_id) as queue: - while True: - event = await queue.get() - await websocket.send_json(event) -``` - -πŸ‘‰ [Event Streaming Reference Documentation](features/ecosystem/stream.md) - ---- - -!!! tip "πŸ’‘ Fast Scaffold Lookup" - Use the `zc startapp -t` command when generating a new domain module to start with a pre-configured template structure, saving manual setup time. \ No newline at end of file +# WebSocket & Redis PubSub + +Multi-subscriber real-time event streaming with cluster-wide broadcasting and automated backpressure management. + +--- + +
+
+ Type
+ Real-time Streaming +
+
+ Status
+ Optional Utility +
+
+ Underlying Tech
+ asyncio.Queue / Redis PubSub +
+
+ +## The Challenge +Scaling WebSockets in a distributed environment (multiple server instances) is complex. If **User A** is connected via WebSocket to **Server 1**, but a relevant event (like a new message) is processed by **Server 2**, Server 2 has no direct way to reach the socket on Server 1. + +Furthermore, managing these connections in raw FastAPI requires: +1. **Manual PubSub Wiring:** Subscribing to Redis channels and correctly parsing messages. +2. **Memory Leak Risks:** Forgetting to unregister listeners when a WebSocket disconnects. +3. **Slow Consumer Protection:** Preventing a slow client from filling up the server's memory with unsent messages. + +## The ZCore Elegance +The `StreamManager` provides a unified interface for local and distributed events. It uses a **Subscription Context Manager** that automatically handles registration, Redis PubSub pattern-matching, and silent cleanup upon disconnection. It implements an internal backpressure policy that automatically evicts "stuck" listeners to preserve system stability. + +=== "ZCore Stream Subscription" + :::python + from zcore.web import StreamManager + from fastapi import WebSocket + + stream_manager = StreamManager() + + @app.websocket("/ws/{user_id}") + async def websocket_endpoint(websocket: WebSocket, user_id: uuid.UUID): + await websocket.accept() + + # 1. Atomic Subscription & Auto-Cleanup + async with stream_manager.subscription(user_id) as queue: + try: + while True: + # 2. Non-blocking retrieval + data = await queue.get() + await websocket.send_json(data) + except WebSocketDisconnect: + pass + # 3. Context manager ensures unsubscription from Redis/Local memory + +=== "FastAPI Raw Implementation" + :::python + # Manual management across multiple server instances + @app.websocket("/ws/{user_id}") + async def websocket_endpoint(websocket: WebSocket, user_id: uuid.UUID): + await websocket.accept() + pubsub = redis.pubsub() + await pubsub.subscribe(f"user:{user_id}") + + try: + async for message in pubsub.listen(): + if message["type"] == "message": + await websocket.send_json(json.loads(message["data"])) + finally: + # 1. Manual cleanup is easy to forget + # 2. No protection against slow consumer backpressure + # 3. No easy way to broadcast to local queues without Redis + await pubsub.unsubscribe(f"user:{user_id}") + +--- + +## Boundaries & Integration +ZCore manages the routing logic, leaving the transport to FastAPI. + +* **Transport Agnostic:** The `StreamManager` provides a standard `asyncio.Queue`. While designed for WebSockets, you can use it for Server-Sent Events (SSE) or long-polling. +* **Redis Optional:** If `init_stream_redis` is not called, the manager operates in "Local-Only" modeβ€”perfect for development or single-node deployments. +* **Structured Serialization:** All messages routed through the manager are automatically serialized/deserialized using ZCore's `CustomJSONEncoder`, supporting UUIDs and Datetime out of the box [utils/helpers.py]. + +--- + +## Under-the-Hood Spec + +### 1. Cluster-Wide Pattern Matching +The `StreamManager` uses `psubscribe` with the pattern `stream:user:*` [web/streams.py]. Instead of creating a unique Redis subscription for every single user (which is expensive), ZCore maintains a single background task (`_listen_to_redis`) that multiplexes incoming cluster-wide messages to the correct local memory queues based on the channel name. + +### 2. Backpressure & Eviction Policy +To prevent memory exhaustion, every user queue has a `maxsize=100` [web/streams.py]. When publishing, ZCore uses `queue.put_nowait()`. If a queue is full (meaning the client is too slow to receive messages), ZCore **evicts the queue immediately** and logs a warning. This "fail-fast" approach protects the server's memory from being consumed by stale or hung connections. + +### 3. Thread-Safe State Management +All modifications to the internal `users_queues` mapping are protected by an `asyncio.Lock` [web/streams.py]. This prevents race conditions where a user might be unsubscribed while a message is simultaneously being routed to them from the Redis background task. + +!!! info "Redis Scalability" + By utilizing the `init_stream_redis` hook, you can share a single Redis connection pool between your Cache, Database (if using Redis), and Stream Manager, optimizing your infrastructure's connection count. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 6e3ae3f..2dd35ad 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,89 +1,288 @@ +
+ ZCore Logo +

+ A pragmatic and complementary architectural layer built on top of FastAPI. + Standardize your structure, protect your data, and manage atomic transactionsβ€”without losing your development freedom. +

+ +
+ pip install fastapi-zcore-framework[all] +
+
-

- ZCore Logo -
-

+--- -# Welcome to ZCore +## From Chaos to Structure -ZCore is not a replacement for FastAPI; it is a modest and practical architectural layer built on top of it. While FastAPI provides the high-performance engine for handling HTTP requests, ZCore provides the "chassis"β€”a structured environment that solves common challenges in medium-to-large scale applications such as dependency management, transaction integrity, and data leakage prevention. +FastAPI is highly performant but structureless, often leading to scattered database sessions, repetitive CRUD boilerplate, and complex dependency wiring in larger teams. -The framework focuses on **Engineered Simplicity**. It abstracts complex patterns like the *Unit of Work* and *Scoped Inversion of Control* into intuitive interfaces, allowing you to focus on your domain logic while the framework ensures that your database transactions are atomic and your sensitive data remains restricted based on the execution context. +ZCore acts as a structured chassis. See how ZCore streamlines standard web operations while keeping you fully in control: ---- +=== "ZCore (35 Lines β€” 7 Secure Endpoints)" + + :::python + from zcore import BaseRouter, BaseService, BaseRepository, Base, Zchema + from sqlalchemy.orm import Mapped, mapped_column + import uuid + + # 1. Define Model & Access Scopes (Auto-generates product:create, product:view, etc.) + class Product(Base): + __tablename__ = "products" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + name: Mapped[str] = mapped_column() + price: Mapped[float] = mapped_column() + secret_cost: Mapped[float | None] = mapped_column() # Sensitive data + + # 2. Context-Aware Schema (Auto-prunes 'secret_cost' in real-time if user lacks permission) + class ProductBase(Zchema): + __model__ = "products" + name: str + price: float + secret_cost: float | None = None + + # 3. Standardize Layers & Scaffold Router (Generates 7 secure CRUD & Search endpoints) + class ProductRepository(BaseRepository[Product, ProductBase, ProductBase]): + def __init__(self, db: AsyncSession): + super().__init__(model=Product, db=db) + + class ProductService(BaseService[Product, ProductBase, ProductBase]): + def __init__(self, repository: ProductRepository = Inject(ProductRepository)): + super().__init__(model=Product, repository=repository) + + class ProductRouter(BaseRouter[ProductBase, ProductBase]): + model = Product + create_schema = ProductBase + update_schema = ProductBase + schema_out = ProductBase + service = ProductService + prefix = "/products" + -## Why Choose ZCore? +=== "FastAPI (100+ Lines β€” Only 3 Basic Endpoints)" -ZCore was designed to bridge the gap between "writing an endpoint" and "building a maintainable system." + :::python + # To replicate just a SUBSET of ZCore's security, transaction, and search features + # in raw FastAPI, you must manually write massive repetitive boilerplate: -| Feature | Standard FastAPI Challenge | The ZCore Approach | -| :--- | :--- | :--- | -| **Dependency Injection** | Manual dependency registration and deeply nested `Depends` functions. | Automated **Scoped IoC** with constructor injection. | -| **Data Security** | Manual filtering of Pydantic models for different users. | **3-Tier Schema Security** via `Zchema` (generation, input, response). | -| **Transactions** | Scatterred `.commit()` calls leading to partial failures. | Centralized **Unit of Work** (UOW) for atomic operations. | -| **Project Structure** | Inconsistent layouts across different teams. | Modular **Plugin System** and standardized CLI scaffolding. | -| **Search & Filter** | Writing repetitive boilerplate for every query. | A secure, dynamic **Search Engine** with depth-limit protection. | + from fastapi import FastAPI, Depends, HTTPException, status + from pydantic import BaseModel + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy import select, func + import uuid + + # 1. Define Base Model + class Product(Base): + __tablename__ = "products" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + name: Mapped[str] = mapped_column() + price: Mapped[float] = mapped_column() + secret_cost: Mapped[float] = mapped_column() + + # 2. Multiple Pydantic models needed to manually prevent data leakage for different roles + class ProductCreate(BaseModel): + name: str + price: float + secret_cost: float + + class ProductOutPublic(BaseModel): + id: uuid.UUID + name: str + price: float + + class ProductOutAdmin(BaseModel): + id: uuid.UUID + name: str + price: float + secret_cost: float + + # 3. Manual Create Endpoint (With manual transaction rollback) + @app.post("/products", response_model=ProductOutAdmin) + async def create_product( + data: ProductCreate, + db: AsyncSession = Depends(get_db), + user: User = Depends(get_current_user) + ): + if "product:create" not in user.scopes: + raise HTTPException(status_code=403, detail="Access denied") + + product = Product(**data.model_dump()) + db.add(product) + try: + await db.commit() # Manual transaction commit + await db.refresh(product) + except Exception: + await db.rollback() # Manual transaction rollback on failure + raise HTTPException(status_code=400, detail="Transaction failed") + return product + + # 4. Manual Get Endpoint (With manual role-based schema projection to prevent leaks) + @app.get("/products/{id}") + async def get_product( + id: uuid.UUID, + db: AsyncSession = Depends(get_db), + user: User = Depends(get_current_user) + ): + result = await db.execute(select(Product).where(Product.id == id)) + product = result.scalar_one_or_none() + if not product: + raise HTTPException(status_code=404, detail="Product not found") + + # Manual permission-based data pruning + if "product:view_sensitive" in user.scopes: + return ProductOutAdmin.model_validate(product) + return ProductOutPublic.model_validate(product) + + # 5. Manual List Endpoint (With manual offset calculations and count queries) + @app.get("/products") + async def list_products( + page: int = 1, + size: int = 20, + db: AsyncSession = Depends(get_db) + ): + offset = (page - 1) * size + count_query = select(func.count()).select_from(Product) + total = (await db.execute(count_query)).scalar_one() + + items_query = select(Product).offset(offset).limit(size) + items = (await db.execute(items_query)).scalars().all() + + return { + "data": [ProductOutPublic.model_validate(i) for i in items], + "meta": {"total": total, "page": page, "size": size} + } --- -## The Request Lifecycle +## Instant Frontend Integration + +One of ZCore's standout features is its native schema projection. By enabling a single flag on your router, you can instantly render dynamic forms on the frontend. + +Adding `?schema=true` to any scaffolded POST, PUT, or PATCH endpoint returns the raw, validated JSON-Schema of that endpoint: -Understanding how a request travels through ZCore is key to mastering its architecture. The following diagram illustrates the automated orchestration from the moment a request hits the server to the final pruned response. - -```mermaid -sequenceDiagram - participant C as Client - participant M as Middlewares - participant R as ZCoreAPIRoute - participant S as Service & UOW - participant D as Database - participant P as Zchema Security Layer - - C->>M: HTTP Request - Note over M: Generate Correlation ID
Initialize Scoped DI - M->>R: Route Handled - Note over R: Schema Analysis
Restricted Field Lookup - R->>S: Invoke Business Logic - S->>D: DB Operations (via Repository) - Note over S: Atomic Commit via Unit of Work - D-->>S: Record Persisted - S-->>R: Return Domain Model - R->>P: Native Zchema Serialization - Note over P: Prune Restricted Fields
(e.g., user.email, employee.salary) - P-->>R: Pruned Data (via model_serializer hook) - R-->>C: JSON Response (Pruned) +```bash +GET /products?schema=true ``` -πŸ’‘ **Note:** `ZCoreAPIRoute` acts as a smart gateway, automatically intercepting requests and responses to handle schema inspection and dynamic field pruning β€” all delegated to the `Zchema` base class's native Pydantic V2 hooks. +```json +{ + "success": true, + "message": "Schema generated successfully", + "data": { + "title": "ProductBase", + "type": "object", + "properties": { + "name": { "type": "string" }, + "price": { "type": "number" } + }, + "required": ["name", "price"] + }, + "meta": { + "restricted_fields": [] + } +} +``` + +!!! info "Security Isolation" + If the active user does not have permission to view or edit specific fields, ZCore automatically prunes those fields from the returned JSON-Schema in real-time. --- -## Core Pillars at a Glance +## The Request Lifecycle + +ZCore coordinates your web, context, database, and transaction layers into a single, predictable journey. + +

+ The Request Lifecycle +

-* **⚑ Scoped IoC Container:** Manage object lifecycles (Singleton, Transient, or Scoped) with ease. Scoped dependencies are automatically cleared at the end of every HTTP request to prevent memory pollution. -* **πŸ›‘οΈ Secure Search Engine:** A dynamic query builder that supports nested filters and eager-loading, while automatically blocking access to restricted database columns based on security policies. -* **πŸ”— Unit of Work (UOW):** Ensures that business operations succeed or fail as a single unit. It coordinates database flushes and delays event dispatching until the transaction is successfully committed. -* **πŸ—οΈ Modular Plugin System:** Organize your application into decoupled domains. Each plugin manages its own lifecycle hooks (`on_startup`, `on_shutdown`) and can declare dependencies on other plugins. --- -## Where to Start? +## Six Architectural Pillars + +ZCore packages essential enterprise-grade patterns into optional, lightweight modules. + +
+ +
+
πŸ›‘οΈ
+

Context Shielding (Zchema)

+

+ Write a single schema model. ZCore dynamically prunes input fields (preventing mass assignment) and output fields (preventing data leakage) based on active user context. +

+
-Choose the path that best fits your current needs: +
+
πŸ”—
+

Atomic Transactions (UOW)

+

+ Group multiple operations into a single Unit of Work. Domain events are queued and only dispatched after the database transaction commits successfully. +

+
-!!! info "πŸš€ Quick Start" - New to ZCore? Learn how to initialize your first project and create a modular app using our CLI tool in under 5 minutes. - [View Quick Start Guide](learn/overview.md) +
+
πŸ”
+

Dynamic Search Engine

+

+ A secure query builder that translates nested JSON filters, sorting, and eager-loading into safe SQL queries with depth-limit protection against DoS. +

+
-!!! tip "🧠 Architectural Deep Dive" - Want to understand the "Under the Hood" mechanics? Explore our detailed documentation on Scoped DI, UOW, and the Kernel. - [Explore Core Concepts](features/infrastructure/di/) +
+
πŸ—οΈ
+

Scoped DI Container

+

+ Clean constructor-based dependency injection. Registers singletons, transients, and request-scoped dependencies that are auto-cleaned per request. +

+
-!!! warning "πŸ“œ Cheat Sheet" - Already familiar with ZCore? Use our quick reference for syntax, CLI commands, and standard repository methods. - [Open Cheat Sheet](cheatsheet.md) +
+
πŸš€
+

Progressive Routing

+

+ Scaffold 7 CRUD and search endpoints in seconds, or bypass the router completely to write standard, raw FastAPI functional endpoints. +

+
+ +
+
πŸ“¦
+

Topological Plugins

+

+ Structure your codebase into clean modules. The central kernel automatically sorts and loads plugins based on their declared dependencies. +

+
+ +
--- -

- ZCore is licensed under the Apache License 2.0. Built with β˜• and architectural rigor. -

\ No newline at end of file + +## Built for Absolute Freedom + +Is ZCore another rigid framework like Django? **No.** + +We believe in **progressive disclosure**. You only use what you need, when you need it. If a specific business route requires raw, custom logic, bypass the scaffolding router entirely while still using ZCore's decoupled background layers: + +```python +# Bypass the scaffolding BaseRouter and write standard, functional FastAPI +@app.post("/custom-checkout") +async def checkout( + data: CheckoutRequest, + checkout_service: CheckoutService = Inject(CheckoutService) # Still uses Scoped DI +): + # Execute transactional business logic + return await checkout_service.process_order(data) +``` + +!!! success "Next Steps" + Ready to experience clean FastAPI architecture? Proceed to the [Installation & Scaffolding](learn/installation/) guide to create your first ZCore app in seconds. \ No newline at end of file diff --git a/docs/learn/1-db.md b/docs/learn/1-db.md index 6b1cd08..8984162 100644 --- a/docs/learn/1-db.md +++ b/docs/learn/1-db.md @@ -1,72 +1,79 @@ -# πŸ—„οΈ Step 1: Database Models - -Following our bottom-up development flow, we start at the physical database representation. In ZCore, database models are defined using asynchronous SQLAlchemy 2.0 declarative mappers. - -Open `products/models.py` and replace its contents with the following: - -```python -import uuid -from sqlalchemy import String, Integer, Numeric -from sqlalchemy.orm import Mapped, mapped_column -from zcore.db.setup import Base - -class Product(Base): - """Database representation of our Product catalog.""" - - __tablename__ = "products" - - id: Mapped[uuid.UUID] = mapped_column( - primary_key=True, - default=uuid.uuid4 - ) - name: Mapped[str] = mapped_column( - String(120), - nullable=False, - index=True - ) - price: Mapped[float] = mapped_column( - Numeric(10, 2), - nullable=False, - default=0.0 - ) - stock: Mapped[int] = mapped_column( - Integer, - nullable=False, - default=0 - ) -``` +# Database Models & Metadata ---- - -## πŸ“Š Database Column Schema +Define your schema and auto-generate standardized permission scopes for every entity. -For reference, here is how our `Product` Python model maps directly to the underlying SQL database columns: +--- -| Python Property | SQL Column Type | Nullable | Default / Behavior | -| :--- | :--- | :--- | :--- | -| πŸ†” `id` | `UUID` | ❌ No | Primary Key, Auto-generated UUIDv4 | -| πŸ“ `name` | `VARCHAR(120)` | ❌ No | Indexed for high-performance searches | -| πŸ’° `price` | `NUMERIC(10, 2)` | ❌ No | `0.0` (Uses precise decimal scale) | -| πŸ“¦ `stock` | `INTEGER` | ❌ No | `0` (Represents physical inventory) | +
+
+ Type
+ Declarative Base +
+
+ Status
+ Core Requirement +
+
+ Underlying Tech
+ SQLAlchemy 2.0 Async +
+
+ +## The Challenge +In large-scale FastAPI applications, managing authorization often becomes a "string-guessing" game. Developers manually define permission strings like `"product:view"` or `"order:delete"` in routers, while the models remain unaware of these scopes. This leads to **permission drift**, where a table is renamed or a scope is mistyped, resulting in silent authorization failures or security loopholes. + +## The ZCore Elegance +ZCore's `Base` is a thin extension of SQLAlchemy's `DeclarativeBase`. It provides a centralized `.actions()` method that dynamically generates a structured `Actions` object based on the `__tablename__`. These actions are then used by ZCore routers and security layers to enforce consistent authorization without hardcoded strings. + +=== "ZCore Model with Actions" + :::python + from zcore import Base + from sqlalchemy.orm import Mapped, mapped_column + import uuid + + class Product(Base): + __tablename__ = "products" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + name: Mapped[str] = mapped_column() + + # ZCore automatically provides: + # Product.actions().CREATE -> "products:create" + # Product.actions().VIEW -> "products:view" + # Product.actions().DELETE -> "products:delete" + # Product.actions().LISTVIEW -> "products:listview" + +=== "Manual SQLAlchemy Approach" + :::python + # Standard SQLAlchemy models don't know about permissions. + # You are forced to manually manage strings elsewhere: + class Product(Base): + __tablename__ = "products" + id: Mapped[uuid.UUID] = mapped_column(primary_key=True) + + # In your security layer: + PRODUCT_CREATE = "products:create" # Manual string + PRODUCT_VIEW = "products:view" # Manual string + # High risk of typos and zero link to the actual model class. --- -## πŸ›‘οΈ Why ZCore Recommends UUIDs +## Boundaries & Integration +ZCore does not hide SQLAlchemy; it empowers it. -!!! info "πŸ›‘οΈ Distributed Identifier Security" - ZCore uses `uuid.UUID` (specifically UUIDv4) as the default primary key instead of auto-incrementing integers. - Sequential integer IDs (`1, 2, 3...`) expose business metrics (e.g., total products or orders) through simple iteration, and are vulnerable to malicious URL scanning. UUIDs provide distributed identifier safety, allowing you to generate safe IDs client-side or on decoupled nodes without risk of primary key collisions. +* **Standard SQLA 2.0:** Because `Base` inherits from `DeclarativeBase`, you use the modern `Mapped` and `mapped_column` syntax exactly as documented in SQLAlchemy 2.0. +* **Bypass:** If you do not wish to use ZCore's authorization system, you can still use the `Base` class for standard migrations and queries without ever calling `.actions()`. +* **Custom Engines:** While `db_manager` provides a pre-configured `AsyncEngine`, you can still create your own SQLAlchemy engine and bind ZCore models to itβ€”they are 100% compatible with standard `AsyncSession` operations. --- -## πŸ“‹ Declaring Table Metadata +## Under-the-Hood Spec -By inheriting from `Base`, the `Product` model is automatically registered with the metadata engine. This allows it to work with SQLAlchemy's async migration patterns and gives it access to ZCore's declarative permission mappings: +### 1. The `Actions` Dataclass +The `Actions` object is an immutable, frozen dataclass [setup.py]. When `Base.actions()` is called, ZCore iterates over the internal fields of the `Actions` class and interpolates the `__tablename__` to create unique, namespaced permission keys. This ensures that permission names are always derived directly from the source of truthβ€”the database schema. -```python -# ZCore automatically generates table permission actions: -# Product.actions().VIEW -> "products:view" -# Product.actions().CREATE -> "products:create" -``` +### 2. Intelligent Connection Pooling +The `DatabaseManager.init_app` method contains specific logic for SQLite vs. server-based databases (PostgreSQL/MySQL) [setup.py]. For SQLite, it disables `pool_size` and `max_overflow` to prevent locking issues common with file-based databases, while enabling high-performance connection pooling and `pool_pre_ping` for production-grade relational servers. -In the next step, we will define the Pydantic schemas that validate data payload boundaries for these model fields. \ No newline at end of file +### 3. Lifecycle Awareness +ZCore models are designed to be used with the `db_manager.session()` context manager. This manager ensures that if an exception occurs during a database operation, `session.rollback()` is invoked automatically before the error is propagated, keeping your database state clean without manual `try/except` blocks in every service. \ No newline at end of file diff --git a/docs/learn/2-schemas.md b/docs/learn/2-schemas.md index 35f90fe..fb8ac78 100644 --- a/docs/learn/2-schemas.md +++ b/docs/learn/2-schemas.md @@ -1,87 +1,92 @@ -# πŸ“‹ Step 2: Pydantic Schemas +# Context-Aware Data Shielding -While the database model represents how data is **stored**, Pydantic schemas represent how data is **transported**. This modest separation ensures that internal database logic remains decoupled from the external API contract, providing a safety buffer for your data. - -Open `products/schemas.py` and define the following contracts: - -```python -import uuid -from decimal import Decimal -from typing import Optional -from pydantic import Field, ConfigDict -from zcore import Zchema - -class ProductBase(Zchema): - """Shared attributes validated across incoming and outgoing product requests.""" - __db_name__ = "product" - name: str = Field(..., min_length=2, max_length=120, description="Product retail name") - price: Decimal = Field(..., ge=0.00, max_digits=10, decimal_places=2, description="Product price") - stock: int = Field(default=0, ge=0, description="Available stock inventory") - -class ProductCreate(ProductBase): - """Schema for validating new product creation payloads.""" - pass - -class ProductUpdate(Zchema): - """Schema for validating product update payloads (supports partial patches).""" - __db_name__ = "product" - name: Optional[str] = Field(None, min_length=2, max_length=120) - price: Optional[Decimal] = Field(None, ge=0.00, max_digits=10, decimal_places=2) - stock: Optional[int] = Field(None, ge=0) - -class ProductResponse(ProductBase): - """Schema representing the serialized API response payload.""" - id: uuid.UUID - - model_config = ConfigDict(from_attributes=True) -``` +Define one schema; ZCore handles input pruning and output masking based on active security context. --- -## πŸ› οΈ Schema Responsibility Matrix - -To help understand when to use each schema, refer to this breakdown: - -| Schema Name | Direction | Primary Responsibility | -| :--- | :--- | :--- | -| πŸ“¦ `ProductCreate` | **Inbound** | Validates that name, price, and stock exist and meet constraints. | -| πŸ› οΈ `ProductUpdate` | **Inbound** | Allows clients to update only specific fields (Optionality). | -| πŸ“€ `ProductResponse` | **Outbound** | Formats the data for the client and includes the generated `id`. | +
+
+ Type
+ Schema Wrapper +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ Pydantic V2 / ContextVars +
+
+ +## The Challenge +Modern APIs often suffer from two major data integrity issues: +1. **Mass Assignment:** An attacker sends `{"is_admin": true}` in a registration payload, and the backend blindly persists it. +2. **Data Leakage:** Sensitive fields (like `internal_notes` or `secret_cost`) are accidentally included in a public GET response. + +To solve this in raw FastAPI, developers typically create a "Schema Explosion"β€”multiple Pydantic models for every single operation: `UserCreate`, `UserUpdate`, `UserPublicResponse`, `UserAdminResponse`, and `UserOwnerResponse`. This leads to massive maintenance overhead and fragile code. + +## The ZCore Elegance +The `Zchema` class eliminates schema explosion. By linking a schema to its database `__model__` name, ZCore automatically cross-references the active `restricted_fields` in the current context. If a field is restricted, ZCore silently prunes it from incoming payloads and masks it in outgoing responsesβ€”all using a single model definition. + +=== "ZCore Unified Zchema" + :::python + from zcore import Zchema + + class ProductSchema(Zchema): + __model__ = "products" # Links to Product.__tablename__ + + name: str + price: float + secret_cost: float | None = None # Restricted field + + # If "products.secret_cost" is in restricted_fields: + # 1. POST input: 'secret_cost' is silently stripped. + # 2. GET response: 'secret_cost' is masked from JSON output. + # 3. ?schema=true: 'secret_cost' is removed from JSON-Schema. + +=== "FastAPI Schema Explosion" + :::python + # You are forced to maintain 3+ models for the same entity + class ProductBase(BaseModel): + name: str + price: float + + class ProductCreate(ProductBase): + pass # No secret_cost allowed here + + class ProductResponsePublic(ProductBase): + id: uuid.UUID + + class ProductResponseAdmin(ProductResponsePublic): + secret_cost: float # Only for admins + + # In your router, you must manually select the right class: + if user.is_admin: + return ProductResponseAdmin.model_validate(product) + return ProductResponsePublic.model_validate(product) --- -## πŸ” The `Zchema` Base Class - -All schemas in ZCore should inherit from `Zchema` (not directly from Pydantic's `BaseModel`). `Zchema` is a drop-in replacement that adds three layers of automatic security: - -| Tier | Mechanism | What It Protects | -| :--- | :--- | :--- | -| **1. Dynamic Schema Generation** | `__get_pydantic_json_schema__` | Removes restricted fields when clients request `?schema=true`. | -| **2. Input Validation** | `model_validator` | Silently strips restricted fields from incoming request payloads (prevents Mass Assignment). | -| **3. Response Pruning** | `model_serializer` | Strips restricted fields from the final outgoing JSON response. | - -### 🏷️ The `__db_name__` Attribute +## Boundaries & Integration +ZCore provides security shielding without breaking the Pydantic ecosystem. -Every `Zchema` subclass must define a `__db_name__` class attribute that binds the schema to its database domain (e.g., `"product"`, `"user"`, `"order"`). This enables the security layer to resolve restricted field paths like `product.price` or `user.email` without namespace collisions across different modules. - -```python -class ProductBase(Zchema): - __db_name__ = "product" # <-- domain binding - name: str -``` +* **Pydantic V2 Native:** `Zchema` inherits from `pydantic.BaseModel`. All Pydantic features like `Field`, `computed_field`, and `model_validator` work out of the box. +* **Automatic Middleware Integration:** Pruning only works if the `ScopedDependencyMiddleware` and `request_context` are active. If used outside a ZCore request, it behaves exactly like a standard Pydantic model. +* **Bypass:** If you need a schema that never prunes data regardless of context, simply inherit from `pydantic.BaseModel` instead of `Zchema`. --- -## πŸ’° Engineering Note: Why `Decimal`? - -!!! tip "πŸ’‘ Precision in Financial Data" - In ZCore, we prefer `decimal.Decimal` over `float` for monetary values like `price`. Floating-point arithmetic can lead to rounding errors (e.g., `0.1 + 0.2` not equaling `0.3`), which is unacceptable in financial systems. Pydantic handles the conversion from JSON numbers to Python Decimals transparently. +## Under-the-Hood Spec ---- +### 1. The `model_serializer` Wrap +ZCore uses the Pydantic V2 `model_serializer(mode="wrap")` hook [projection.py]. When a response is being generated, ZCore intercepts the dictionary representation, identifies the relative paths of restricted fields for the specific `__model__`, and recursively removes them. This happens at the last possible moment before serialization, ensuring zero data leakage. -## πŸ”— Integration with SQLAlchemy +### 2. Silent Input Pruning +The `model_validator(mode="before")` hook [projection.py] acts as a guard against Mass Assignment. Before data is even validated against the model's types, ZCore prunes any keys that the active user is not allowed to touch. This prevents unauthorized fields from ever reaching your service or repository layers. -!!! note "πŸ›‘οΈ The Role of `from_attributes`" - In Pydantic V2, setting `model_config = ConfigDict(from_attributes=True)` enables the schema to parse data directly from database models. Instead of manually mapping SQLAlchemy fields to dictionary keys, Pydantic reads properties directly from the ORM object, even when attributes are loaded lazily. +### 3. Recursive Dot-Path Resolution +Restriction paths can be specific or broad. For example, if `restricted_fields` contains `products.category.internal_id`, ZCore's internal `_prune_data` method will traverse nested dictionaries or lists within the `Product` schema to find and remove that specific deeply nested key [projection.py]. -Now that our data layers (Database and Schema) are defined, we will create the **Repository Layer** to handle SQL operations for us. \ No newline at end of file +!!! info "Security Isolation" + ZCore's pruning logic is strictly read-only regarding the original data object; it operates on a copy during serialization to ensure the internal application state remains intact while the external representation is secured. \ No newline at end of file diff --git a/docs/learn/3-repository.md b/docs/learn/3-repository.md index e9bbc34..21c0511 100644 --- a/docs/learn/3-repository.md +++ b/docs/learn/3-repository.md @@ -1,60 +1,93 @@ -# πŸ›οΈ Step 3: The Repository Layer +# Repository Pattern & Persistence -The repository layer acts as a modest bridge between your database and your business logic. In ZCore, we use the **Repository Pattern** to encapsulate data access. This ensures that your SQL queries are consistent, reusable, and isolated from the rest of your application. - -Open `products/repositories.py` and configure the repository: - -```python -from zcore.db.repository import BaseRepository -from zcore.db.setup import SessionDep - -from .models import Product -from .schemas import ProductCreate, ProductUpdate - -class ProductRepository(BaseRepository[Product, ProductCreate, ProductUpdate]): - """Data Access Repository managing SQL operations for Products.""" - - def __init__(self, db: SessionDep): - # We pass the declarative model and the active async database session - # to the parent BaseRepository class - super().__init__(model=Product, db=db) -``` +Decouple business logic from database execution with high-performance, type-safe, and reusable data interfaces. --- -## πŸ› οΈ Instant CRUD Capabilities - -By inheriting from `BaseRepository`, your `ProductRepository` immediately gains a set of standard asynchronous methods. You don't need to write raw SQL for common operations: - -| Method | Action | Description | -| :--- | :--- | :--- | -| πŸ” `get(id)` | **Read** | Fetches a single record by its UUID. | -| πŸ“š `get_list()` | **Read** | Retrieves records with built-in pagination support. | -| ✨ `create(schema)` | **Write** | Validates the input and persists a new product. | -| πŸ› οΈ `update(id, schema)` | **Write** | Safely applies changes to an existing record. | -| πŸ—‘οΈ `delete(id)` | **Write** | Removes a record from the database. | +
+
+ Type
+ Data Access Layer +
+
+ Status
+ Recommended +
+
+ Underlying Tech
+ SQLAlchemy 2.0 / Pydantic V2 +
+
+ +## The Challenge +In many FastAPI projects, database queries are scattered directly inside route handlers. This "Fat Router" anti-pattern leads to: +1. **Repetitive Boilerplate:** Manually writing `session.execute(select(...))` and `.scalars().all()` for every endpoint. +2. **Brittle Pagination:** Inconsistent implementations of offset or cursor logic across different domains. +3. **Testing Hardship:** Difficulty mocking database interactions because the business logic is tightly coupled with SQLAlchemy's execution engine. +4. **Inefficient Queries:** Forgetting to apply `load_only` or missing eager-loading configurations, leading to over-fetching or N+1 problems. + +## The ZCore Elegance +The `BaseRepository` provides a robust, pre-optimized set of CRUD and search operations. It automatically inspects your model to identify primary keys and coordinates with ZCore's pagination and search engines. By inheriting from `BaseRepository`, you gain standardized behaviors for selective field loading, batch operations, and partial updates. + +=== "ZCore Repository Interface" + :::python + from zcore import BaseRepository + from .models import Product + from .schemas import ProductCreate, ProductUpdate + + class ProductRepository(BaseRepository[Product, ProductCreate, ProductUpdate]): + def __init__(self, db: AsyncSession): + # No boilerplate required. BaseRepository handles: + # .get(), .get_list(), .search(), exist(), get_by_ids() + # .create(), .create_multi(), .update(), .update_multi() + # .delete(), .delete_multi() + super().__init__(model=Product, db=db) + + async def get_expensive_products(self, threshold: float): + # Custom logic is clean and isolated + query = self._get_base_query().where(Product.price > threshold) + result = await self.db.execute(query) + return result.scalars().all() + +=== "Manual SQLAlchemy Boilerplate" + :::python + # For every model, you must manually repeat: + async def get_product(db: AsyncSession, id: uuid.UUID): + query = select(Product).where(Product.id == id) + result = await db.execute(query) + item = result.scalar_one_or_none() + if not item: return None + return item + + async def update_product(db: AsyncSession, id: uuid.UUID, data: dict): + item = await get_product(db, id) + for key, value in data.items(): + setattr(item, key, value) + await db.flush() # Remember to flush but not commit? + return item --- -## πŸ“ Data Flow Visualization +## Boundaries & Integration +ZCore repositories are abstractions that expose, rather than hide, the power of SQLAlchemy. -The repository layer handles the translation between your Pydantic schemas and the SQLAlchemy models, keeping the communication between layers clean and predictable. - -```mermaid -flowchart LR - Service[Service Layer] -->|Calls CRUD Method| Repo[ProductRepository] - Repo -->|Abstraction Layer| SQL[Async SQL Generation] - SQL -->|Executes| DB[(SQLite / PostgreSQL)] -``` +* **Direct DB Access:** The `self.db` attribute is a standard SQLAlchemy `AsyncSession`. You can run raw SQL or complex `expression` constructs whenever needed. +* **Query Customization:** Use `_get_base_query()` to start every query with default filters (e.g., Soft Delete logic) or `_extend_query()` to apply global modifications. +* **Eager Loading:** All read methods (`get`, `get_list`, `get_by_ids`) accept an `options` parameter where you can pass standard SQLAlchemy loader options like `joinedload` or `selectinload`. +* **Selective Fields:** The `fields` parameter allows you to pass a list of model attributes to perform a `load_only` query, optimizing network traffic between the app and the database. --- -## πŸ’‘ Engineering Insights +## Under-the-Hood Spec + +### 1. Automatic PK Inspection +Upon initialization, `BaseRepository` utilizes the SQLAlchemy `inspect()` module to automatically locate the model's primary key [repository.py]. It maps this to `self.pk` and `self.pk_name`, allowing generic CRUD methods to function correctly regardless of whether your primary key is named `id`, `uuid`, or `pk`. -!!! tip "πŸ’‘ Reducing Boilerplate" - The goal of the `BaseRepository` is to remove the "noise" of repetitive SQL code. Unless your application requires complex joins or specialized raw queries, this single file is often all you need for data management. +### 2. The `get_by_ids` Fast-Path +The `get_by_ids` method includes a silent performance optimization: if an empty list is passed as the `ids` argument, the repository returns an empty sequence immediately without making a network round-trip to the database [repository.py]. -!!! info "πŸ›‘οΈ Type Safety" - Because we pass the types `[Product, ProductCreate, ProductUpdate]` to the generic `BaseRepository`, your IDE will provide full auto-completion and type checking for all database operations, reducing runtime errors. +### 3. Flush-Oriented Mutations +All mutation methods (`create`, `update`, `delete`) in the repository utilize `await self.db.flush()` instead of `commit()` [repository.py]. This ensures that changes are sent to the database's transaction buffer but remain uncommitted, allowing the `UnitOfWork` or a higher-level service to manage the final atomic transaction boundary. -Now that our data access is ready, we will build the **Service Layer** to house our business rules. \ No newline at end of file +### 4. Search Engine Integration +The `search()` method is a bridge to the `SearchEngine`. It translates a `SearchRequest` (containing JSON-based filters and sorts) into a validated SQLAlchemy query, ensuring that even dynamic queries respect the security context and depth limits [repository.py, search.py]. \ No newline at end of file diff --git a/docs/learn/4-service.md b/docs/learn/4-service.md index 062ceec..23e4f36 100644 --- a/docs/learn/4-service.md +++ b/docs/learn/4-service.md @@ -1,103 +1,106 @@ -# βš™οΈ Step 4: Business Service Layer - -The service layer is the modest heart of your application. It is where you define your business rules and orchestrate data transactions. While the Repository handles **how** to talk to the database, the Service handles **what** happens to the data before and after it reaches the storage. - -Open `products/services.py` and implement the service logic: - -```python -from zcore.service.base import BaseService -from zcore.kernel.di import Inject -from zcore.exceptions.base import ValidationError - -from .models import Product -from .schemas import ProductCreate, ProductUpdate -from .repositories import ProductRepository - -class ProductService(BaseService[Product, ProductCreate, ProductUpdate]): - """Service layer coordinating business transactions for the Product domain.""" - - def __init__(self, repository: ProductRepository = Inject(ProductRepository)): - # Inject the ProductRepository class dependency using our IoC Container - super().__init__(model=Product, repository=repository) - - async def pre_create(self, schema: ProductCreate) -> None: - """Lifecycle hook executed automatically before a database insert.""" - if "test" in schema.name.lower(): - raise ValidationError( - message="Cannot create products with reserved placeholder names.", - payload={"field": "name", "value": schema.name} - ) - - async def on_create(self, schema: ProductCreate) -> Product: - """Custom database/persistence logic for single-record creation. - - Override this instead of the public `create()` orchestrator. - The orchestrator guarantees that pre/post hooks and the safe commit - boundary are always executed around this method. - """ - return await self.repository.create(schema) - - async def post_create(self, model: Product) -> None: - """Lifecycle hook executed automatically after successful database insert.""" - # Ideal place for side-effects like sending emails or logs - pass -``` +# Business Services & Lifecycle ---- - -## 🚦 Orchestration Flow +Orchestrate domain logic with transactional safety and granular execution hooks. -The service layer acts as the "manager," ensuring that data follows the correct path and validation rules before being persisted. The public orchestrator methods (`create`, `update`, `delete`) follow the **Template Method** pattern: they coordinate the execution of `pre_*` hooks, the customizable `on_*` database action, `post_*` hooks, and the `_safe_commit` boundary β€” in that exact order. +--- -```mermaid -graph TD - Router[Web Router] -->|Calls| Service[ProductService] - subgraph Service Layer - Pre[pre_create Hook] --> On[on_create Business Logic] - On --> Post[post_create Hook] - end - Service -->|Commands| Repo[ProductRepository] - Repo --> DB[(SQL Database)] -``` +
+
+ Type
+ Domain Logic Layer +
+
+ Status
+ Highly Recommended +
+
+ Underlying Tech
+ Python / Unit of Work +
+
+ +## The Challenge +When business logic lives in the router (the "Fat Controller" anti-pattern), applications become impossible to maintain. Developers struggle with: +1. **Leaky Abstractions:** HTTP logic (headers, status codes) mixing with business rules (tax calculations, validation). +2. **Transaction Fragility:** Manually managing `commit()` and `rollback()` in every function. If one operation fails halfway, the database is left in an inconsistent state. +3. **Hook Spaghetti:** Difficulty triggering side effects (like sending an email or clearing a cache) consistently after a database write. +4. **Redundant Validation:** Re-writing existence checks and ownership validation for every endpoint. + +## The ZCore Elegance +The `BaseService` acts as a pure-Python orchestration layer. It wraps repository calls with a rich set of **Lifecycle Hooks** (`pre_create`, `post_update`, etc.) and a smart **Safe Commit** mechanism. This ensures that business rules are executed in the correct order and that transactions are only finalized when all steps succeed. + +=== "ZCore Orchestrated Service" + :::python + from zcore import BaseService, Inject + from .repositories import OrderRepository + from .models import Order + + class OrderService(BaseService[Order, OrderCreate, OrderUpdate]): + def __init__(self, repository: OrderRepository = Inject(OrderRepository)): + super().__init__(model=Order, repository=repository) + + async def pre_create(self, schema: OrderCreate): + # 1. Validation logic before DB hit + if schema.amount <= 0: + raise ValidationError("Invalid amount") + + async def post_create(self, model: Order): + # 2. Trigger side effects after successful write + await email_service.send_receipt(model.user_id) + +=== "FastAPI Manual Logic" + :::python + # Logic scattered in the route handler + @app.post("/orders") + async def create_order(data: OrderCreate, db: AsyncSession = Depends(get_db)): + # 1. Manual Validation + if data.amount <= 0: + raise HTTPException(400, "Invalid amount") + + # 2. Manual DB Logic + new_order = Order(**data.model_dump()) + db.add(new_order) + try: + await db.commit() + except Exception: + await db.rollback() + raise + + # 3. Manual Side Effect + await email_service.send_receipt(new_order.user_id) + return new_order -### πŸ”§ What to Override +--- -| Method | Purpose | -| :--- | :--- | -| `pre_create` / `pre_update` / `pre_delete` | Validation, authorization, or enrichment **before** the database operation. | -| `on_create` / `on_update` / `on_delete` | **Custom database/persistence logic.** Override these instead of the public `create()` / `update()` / `delete()` orchestrators. | -| `post_create` / `post_update` / `post_delete` | Side-effects (logging, events, cache invalidation) **after** a successful database operation. | +## Lifecycle Execution Flow +ZCore services follow a strict execution sequence for every mutation. If any step (including a hook) raises an exception, the entire operation is aborted. -!!! tip "🧠 Why `on_*` Instead of Overriding `create()`?" - Previously, developers overrode `create()` directly, which risked skipping hooks or the safe commit boundary. The new `on_*` methods are the **only** extension points you need. The orchestrator guarantees that `pre_*` hooks run first, your custom logic runs next, `post_*` hooks run after, and the transaction is committed safely β€” every time. +

+ Lifecycle Execution Flow +

--- -## πŸ› οΈ Automated Dependency Injection +## Boundaries & Integration +ZCore services are designed to be framework-agnostic business units. -!!! info "πŸ’‘ The `Inject` Mechanism" - ZCore uses an Inversion of Control (IoC) container to manage dependencies. When you use `Inject(ProductRepository)`, the framework automatically finds the active database session and provides a ready-to-use repository instance. This keeps your code clean and makes testing much easier. +* **HTTP Independence:** Services do not accept `Request` objects or return `JSONResponse`. They deal strictly with Pydantic schemas and SQLAlchemy models, making them easily testable in CLI scripts or background workers. +* **Dependency Injection:** Use the `Inject()` helper to pull in other services (e.g., `NotificationService`) or repositories directly into the constructor. +* **Bypass:** If you don't need the lifecycle hooks, you can call the core execution methods (e.g., `on_create`, `on_update`) directly, or bypass the service layer entirely by injecting the repository into your router. --- -## 🎣 Lifecycle Hooks - -ZCore provides hooks that wrap around standard database operations. These are designed to keep your code organized by separating validation from the actual data save. +## Under-the-Hood Spec -| Hook | When it runs | Practical Example | -| :--- | :--- | :--- | -| ✨ `pre_create` | Before SQL Insert | Check for name duplicates or validate business rules. | -| βœ… `on_create` | Core database insert | Override to customize the creation logic. | -| βœ… `post_create` | After SQL Insert | Increment user stats or notify other systems. | -| πŸ› οΈ `pre_update` | Before SQL Update | Verify if the product is not "locked" for edits. | -| πŸ”„ `on_update` | Core database update | Override to customize the update logic. | -| πŸ”„ `post_update` | After SQL Update | Clear local cache records for this product. | -| πŸ—‘οΈ `pre_delete` | Before SQL Delete | Ensure the product is not linked to active orders. | -| πŸ—‘οΈ `on_delete` | Core database delete | Override to customize the deletion logic. | +### 1. The Safe Commit Protocol +The most critical feature of `WriteServiceMixin` is `_safe_commit()` [service/base.py]. Before committing, the service inspects the database session's internal metadata (`session.info`). If the key `uow_managed` is set to `True`, the service **refrains from committing**. This allows an outer `UnitOfWork` to group multiple service calls into a single atomic transaction without the services "breaking" the transaction boundary prematurely. ---- +### 2. Post-Read Interceptors +Read operations (`get`, `get_list`) include `post_get` and `post_get_multi` hooks [service/base.py]. These are ideal for dynamic property calculation or permission-based filtering that happens after the data leaves the repository but before it reaches the router. -!!! tip "🧠 Engineering Note: Transaction Safety" - All write operations inside a service are handled atomically. If an exception is raised in `pre_create` or even during the `on_create` database action, ZCore will automatically roll back the transaction, ensuring your database remains in a consistent state. +### 3. Automated Error Mapping +The service layer automatically translates missing database records into standardized `EntityNotFound` exceptions [service/base.py]. This ensures that your API consistently returns a `404 Not Found` with a structured error message without you having to write manual `if not result` checks in every method. -In the next step, we will expose this service to the outside world using the **Web Router**. \ No newline at end of file +### 4. Partial Patch Support +The `update` method intelligently handles the `partial` flag [service/base.py]. When `partial=True`, it instructs the repository to ignore `None` or unset values in the update schema, facilitating clean HTTP `PATCH` implementations. \ No newline at end of file diff --git a/docs/learn/5-router.md b/docs/learn/5-router.md index 1bc86bd..8224a78 100644 --- a/docs/learn/5-router.md +++ b/docs/learn/5-router.md @@ -1,138 +1,91 @@ -# 🌐 Step 5: Automated Web Routing - -The routing layer is the final modest bridge that connects your business logic to the internet. ZCore's `BaseRouter` provides a standard way to scaffold common CRUD operations (POST, GET, PUT, PATCH, DELETE), ensuring that your API follows consistent patterns while handling security and pagination automatically. - -Open `products/routers.py` and set up the web layer: - -```python -from typing import Any -from zcore.web.base_router import BaseRouter, RouteKey -from zcore.db.pagination import PageNumberPagination - -from .schemas import ProductCreate, ProductUpdate, ProductResponse -from .services import ProductService -from .models import Product - -class ProductRouter(BaseRouter[ProductCreate, ProductUpdate]): - """Scaffolded router exposing standard web operations for Products.""" - - model = Product - create_schema = ProductCreate - update_schema = ProductUpdate - schema_out = ProductResponse - service = ProductService - - # Configure path context - prefix = "/products" - tags = ["Product Management"] - - # Expose raw JSON schemas to API clients (via ?schema=true) - expose_schemas = True - - # Use standard offset-based page-number pagination - pagination_class = PageNumberPagination - -# Export the active router instance for registration -router_instance = ProductRouter() -``` +# Progressive API Scaffolding ---- - -## πŸ› οΈ Scaffolded Endpoints Reference - -By inheriting from `BaseRouter`, your application immediately provides the following endpoints. ZCore maps these to fine-grained security scopes based on the model's metadata: - -| Method | Endpoint | Security Scope | Description | -| :--- | :--- | :--- | :--- | -| πŸ†• `POST` | `/products/` | `products:create` | Creates a new product record. | -| πŸ” `GET` | `/products/{id}` | `products:view` | Retrieves details of a specific product. | -| πŸ“š `GET` | `/products/` | `products:listview` | Lists products with pagination. | -| πŸ§ͺ `POST` | `/products/search` | `products:listview` | Advanced filtering and searching. | -| πŸ“ `PUT` | `/products/{id}` | `products:update` | Full update of an existing product. | -| 🩹 `PATCH` | `/products/{id}` | `products:update` | Partial update (patch) of specific fields. | -| πŸ—‘οΈ `DELETE` | `/products/{id}` | `products:delete` | Removes a product from the database. | +Generate 7 secure CRUD and search endpoints in seconds with zero manual boilerplate. --- -## 🎯 Unified Route Dependencies - -ZCore has replaced the old static permission properties (`POST_PERMISSIONS`, `GET_PERMISSIONS`, etc.) with a single, elegant OOP method: `get_route_dependencies`. - -This method receives the `RouteKey` and the computed `action` string, and returns a list of FastAPI dependencies. By default, it returns a `HasScopes(action)` dependency, which enforces the appropriate security scope. - -### πŸ”§ Customizing Dependencies for Specific Routes - -To override dependencies for a specific HTTP method, simply override `get_route_dependencies` and use `super()` to fall back to the default behavior for other routes: - -```python -from typing import Any -from zcore.web.base_router import BaseRouter, RouteKey - -class ProductRouter(BaseRouter[ProductCreate, ProductUpdate]): - model = Product - # ... other config - - def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: - """Override route dependencies with custom logic.""" - if route_key == RouteKey.DELETE: - # Only admins can delete products - return [Depends(AdminOnly())] +
+
+ Type
+ Web Router Scaffolding +
+
+ Status
+ Optional Utility +
+
+ Underlying Tech
+ FastAPI / Starlette +
+
+ +## The Challenge +Writing standard CRUD (Create, Read, Update, Delete) routes in FastAPI is repetitive. For every new domain, developers must: +1. Define 5-7 path functions. +2. Manually inject the correct service. +3. Apply standard status codes (`201 Created` vs `200 OK`). +4. Wrap every response in a consistent envelope. +5. Manually apply security dependencies (scopes) that often fall out of sync with the database model permissions. + +## The ZCore Elegance +The `BaseRouter` automates the scaffolding of a fully secure API. It maps the standard HTTP methods to your business service and automatically derives security scopes from your model's `.actions()`. It supports dynamic sorting, filtering (via the Search engine), and pagination out of the box. + +=== "ZCore Scaffolding" + :::python + from zcore import BaseRouter, RouteKey + from .schemas import ProductCreate, ProductUpdate, ProductResponse + from .services import ProductService + from .models import Product + + class ProductRouter(BaseRouter[ProductCreate, ProductUpdate]): + model = Product + create_schema = ProductCreate + update_schema = ProductUpdate + schema_out = ProductResponse + service = ProductService + + prefix = "/products" + tags = ["Products"] + expose_schemas = True # Enables ?schema=true + +=== "FastAPI Raw Scaffolding" + :::python + # To achieve the same result, you must write ~80 lines of code: + router = APIRouter(prefix="/products", tags=["Products"]) + + @router.post("/", status_code=201, response_model=ResponseWrapper[ProductResponse]) + async def create(data: ProductCreate, service: ProductService = Depends(get_product_service)): + if "products:create" not in user.scopes: # Manual scope check + raise ForbiddenError() + result = await service.create(data) + return ResponseWrapper(data=result) - # Fall back to standard scope-based dependencies for all other routes - return super().get_route_dependencies(route_key, action) -``` - -!!! tip "🧠 Why `super()` Delegation?" - By calling `super().get_route_dependencies(route_key, action)`, you inherit ZCore's automatic action-to-scope mapping for every route except the one you're customizing. This keeps your code DRY and ensures that new endpoints added in future versions of ZCore are automatically secured. - -### πŸ“‹ Available `RouteKey` Values - -| RouteKey | HTTP Method | Default Action | -| :--- | :--- | :--- | -| `RouteKey.POST` | `POST` | `{model}:create` | -| `RouteKey.GET` | `GET` | `{model}:view` | -| `RouteKey.GET_ALL` | `GET (list)` | `{model}:listview` | -| `RouteKey.SEARCH` | `POST /search` | `{model}:listview` | -| `RouteKey.UPDATE` | `PUT` | `{model}:update` | -| `RouteKey.PATCH` | `PATCH` | `{model}:update` | -| `RouteKey.DELETE` | `DELETE` | `{model}:delete` | + # ... Repeat for GET, GET ALL, SEARCH, PUT, PATCH, DELETE ... + # (And don't forget to handle pagination and 404s manually) --- -## 🚦 Request Execution Flow - -The router doesn't just pass data; it acts as a security guard, ensuring the user has the correct scopes before any business logic is executed. +## Boundaries & Integration +ZCore routers provide "Progressive Scaffolding"β€”they grow with your complexity without locking you in. -```mermaid -flowchart TD - Client[Client Request] -->|GET /products/| Router[ProductRouter] - Router -->|Resolve Dependencies| Deps["get_route_dependencies(RouteKey.GET, 'products:listview')"] - Deps -->|Returns HasScopes| Scope["HasScopes('products:listview')"] - Scope -->|Validation Success| Service[ProductService] - Service -->|Fetch Data| Result[Product List] - Result -->|Wrap Response| Envelope[ResponseWrapper] - Envelope -->|Clean JSON| Client -``` +* **Endpoint Exclusion:** Use the `exclude` attribute to skip scaffolding specific routes. + `exclude = {RouteKey.DELETE, RouteKey.PATCH}` +* **OOP Freedom:** Every scaffolded endpoint is an overridable method (e.g., `create_endpoint`). You can override them to add custom logic while keeping the rest of the router automated. +* **Dynamic Dependencies:** Override `get_route_dependencies` to inject custom authentication, logging, or rate-limiting for specific route keys (e.g., stricter auth for `DELETE`). +* **Standard APIRouter:** The `self.router` attribute is a standard FastAPI `APIRouter`. You can add custom, raw FastAPI endpoints to it at any time using standard decorators. --- -## πŸ’‘ Engineering Insights +## Under-the-Hood Spec -!!! tip "πŸ’‘ Pure OOP, No Magic" - Unlike the old static property cascade (which used complex fallback logic to resolve `DEFAULT_PERMISSIONS` against individual route properties), the new `get_route_dependencies` method is resolved deterministically at startup time using standard Python inheritance and `super()` calls β€” zero runtime reflection overhead. +### 1. Unified Response Class +ZCore routers use the `ZCoreAPIRoute` class by default [web/api_router.py]. This specialized route ensures that all responses use the `ZCoreJSONResponse`, which utilizes the framework's custom JSON encoder to safely handle UUIDs, Datetime, and Decimal types without manual conversion. -!!! info "πŸ›‘οΈ Dynamic Schema Exposure" - By setting `expose_schemas = True`, you allow clients to call your endpoints with the `?schema=true` query parameter. ZCore will return the JSON Schema for that specific endpoint, which is useful for dynamic form generation or automated client-side validation. +### 2. Automated Action Mapping +The router automatically coordinates with your model's `Actions` system [web/base_router.py]. It maps `RouteKey.POST` to `model.actions().CREATE` and `RouteKey.DELETE` to `model.actions().DELETE`. This ensures that your API's security requirements are always physically bound to your database schema. -!!! note "πŸ“¦ Standardized Response Envelope" - Consistency is key for frontend developers. ZCore automatically wraps every response in a `ResponseWrapper`. Your clients will always receive a predictable structure: - ```json - { - "success": true, - "message": "Success", - "data": { ... }, - "meta": { ... } - } - ``` +### 3. Dynamic Schema Extraction +If `expose_schemas` is enabled, the router intercepts requests with the `?schema=true` query parameter [web/api_router.py]. It identifies the target Pydantic model for the current route (whether it's an input model for POST or an output model for GET), generates its JSON-Schema, and prunes it based on the user's active context restrictions before returning it. -Now that our module is complete, we will package it as a reusable **Framework Plugin** in the final step. \ No newline at end of file +### 4. Vary Header Management +To prevent caching issues when field masking is active, the `ZCoreAPIRoute` automatically appends `Authorization` or `Cookie` to the HTTP `Vary` header if restricted fields are detected in the active context [web/api_router.py]. This tells proxies and browsers that the response content may change based on the user's identity. \ No newline at end of file diff --git a/docs/learn/6-plugin.md b/docs/learn/6-plugin.md index 06db791..a7accf6 100644 --- a/docs/learn/6-plugin.md +++ b/docs/learn/6-plugin.md @@ -1,91 +1,97 @@ -# πŸ—οΈ Step 6: Domain Packaging (Plugins) +# Topological Domain Packaging -In ZCore, domain modules are packaged as reusable **Plugins**. Each plugin acts as a modest, self-contained unit that manages its own routing, lifecycle tasks, and internal setup. This ensures that your application remains modular and easy to maintain as it grows. - -Open `products/plugin.py` and define the plugin wrapper: - -```python -from fastapi import FastAPI -from zcore.kernel import Plugin -import structlog - -from .routers import router_instance - -logger = structlog.get_logger() - -class ProductPlugin(Plugin): - """Plugin wrapper packaging the Product Management domain.""" - - name = "product_domain" - version = "0.1.0" - dependencies = [] # List required plugins (e.g. ['security_plugin']) - - def setup(self, app: FastAPI) -> None: - """Register routes with the main FastAPI instance.""" - app.include_router(router_instance.router) - logger.info("Product Router registered successfully.") - - async def before_startup(self) -> None: - """Run operations before the web server starts.""" - logger.info("Product domain: preparing assets...") - - async def on_startup(self) -> None: - """Run core startup logic (e.g. warming up caches).""" - logger.info("Product domain: warming local product memory caches...") - - async def after_startup(self) -> None: - """Run tasks after startup concludes.""" - pass - - async def on_shutdown(self) -> None: - """Clean up resources when the application shuts down.""" - logger.info("Product domain: shutting down connections safely...") -``` +Structure your codebase into clean, self-contained modules with automated dependency resolution. --- -## πŸ“‹ The Plugin Contract - -Every plugin in ZCore follows a simple structure. This allows the core engine (Kernel) to coordinate multiple domains without them interfering with each other. - -| Property / Method | Purpose | -| :--- | :--- | -| 🏷️ `name` | Unique identifier used for dependency resolution. | -| πŸ”’ `version` | Semantic versioning for internal tracking. | -| πŸ”— `dependencies` | A list of other plugin names that must start before this one. | -| πŸ› οΈ `setup()` | Synchronous configuration (Route registration, DI binding). | +
+
+ Type
+ Orchestration Protocol +
+
+ Status
+ Core Framework +
+
+ Underlying Tech
+ graphlib / typing.Protocol +
+
+ +## The Challenge +As projects grow, the `main.py` file often becomes a "garbage bin" for initialization logic. Developers manually wire database connections, cache warmups, and route registrations. This leads to: +1. **Circular Import Hell:** Modules trying to import each other during startup. +2. **Race Conditions:** A service starting before the database connection is fully established. +3. **Spaghetti Entry Points:** Hard-to-read setup code that requires manual ordering (e.g., Auth must load before Billing). +4. **Tightly Coupled Logic:** Inability to easily enable or disable features (like a Search plugin) without hacking the core app setup. + +## The ZCore Elegance +ZCore uses a **Topological Plugin System**. Every domain (Orders, Users, Payments) is packaged as a `Plugin`. Plugins declare their dependencies explicitly. The ZCore Kernel then calculates the mathematically correct startup order using a Directed Acyclic Graph (DAG). This ensures your architecture is decoupled, and every module starts only when its requirements are ready. + +=== "ZCore Modular Plugin" + :::python + from zcore import Plugin + from fastapi import FastAPI + + class InventoryPlugin(Plugin): + name = "inventory" + version = "1.0.0" + dependencies = ["database", "logging"] # Declared requirements + + def setup(self, app: FastAPI): + # Wire routers to the app + from .routers import router + app.include_router(router) + + async def on_startup(self): + # Domain-specific startup logic + await cache_warmer.warm_stock_levels() + + async def on_shutdown(self): + await inventory_client.close() + +=== "FastAPI Manual Sequencing" + :::python + # Standard main.py requires manual, fragile ordering + app = FastAPI() + + @app.on_event("startup") + async def startup(): + # You must manually remember the order: + await init_logging() + await init_db() + # If you move this line up, the app might crash + await cache_warmer.warm_stock_levels() + + app.include_router(inventory_router) + # 500 lines of wiring later... --- -## πŸ”„ Managing Application Lifecycles +## Boundaries & Integration +ZCore Plugins are wrappers, not cages. They act as "Initialization Anchors" for standard FastAPI components. -ZCore provides several "hooks" that allow you to execute code at specific points in the application's life. This is safer than using global variables or loose scripts, as it ensures resources are handled in the correct order. - -```mermaid -graph TD - Start([Server Boot]) --> B[before_startup] - B --> S[on_startup] - S --> A[after_startup] - A --> Running{App Running} - Running --> Shutdown[on_shutdown] - Shutdown --> Stop([Server Stop]) -``` - -| Lifecycle Hook | Ideal Use Case | -| :--- | :--- | -| 🏁 `before_startup` | Pre-flight checks, verifying environment variables. | -| πŸ”₯ `on_startup` | Warming up caches, establishing external connections. | -| ✨ `after_startup` | Diagnostic logging, health-check verification. | -| πŸ›‘ `on_shutdown` | Closing database pools, flushing logs, releasing file locks. | +* **Standard FastAPI Routes:** The `setup(app)` method receives the raw `FastAPI` instance. You use standard `app.include_router()` or `app.add_middleware()` calls exactly as you would in a normal project. +* **Decoupled Lifecycle:** You can write a Plugin that only handles background tasks without having any routes, or a Plugin that only registers Global Middlewares. +* **Manual Bypass:** If a module is too simple to warrant a Plugin, you can still register its routes directly in `main.py` using standard FastAPI methods. However, it will not benefit from ZCore's topological startup ordering. --- -## πŸ’‘ Engineering Insights +## Under-the-Hood Spec + +### 1. Topological Sorting (DAG) +The ZCore Kernel utilizes Python's `graphlib.TopologicalSorter` [kernel/engine.py]. When `kernel.setup(app)` is called, it builds a graph where nodes are plugin names and edges are their `dependencies`. If a cycle is detected (e.g., A depends on B, and B depends on A), ZCore raises a `RuntimeError` immediately at startup to prevent unstable execution. -!!! tip "πŸ’‘ Why Use Plugins?" - Modular design prevents "spaghetti code." By keeping your product logic inside its own plugin, you can easily move it to a different project or convert it into a standalone microservice later with minimal changes. +### 2. Runtime Protocol Validation +The `Plugin` interface is defined as a `runtime_checkable` Protocol [kernel/plugins.py]. This allows ZCore to verify that your classes implement the required attributes (`name`, `version`, `dependencies`) and methods without requiring strict class inheritance, though inheriting from `zcore.Plugin` is recommended for type safety. -!!! info "πŸ›‘οΈ Dependency Resolution" - If your `ProductPlugin` needs a `SecurityPlugin` to be active, simply add `"security_plugin"` to the `dependencies` list. ZCore's Kernel will automatically sort the plugins and ensure the security layer is ready before the product layer starts. +### 3. Sequential Lifecycle Phases +The Kernel executes startup logic in three distinct, sequential waves across all sorted plugins [kernel/engine.py]: +1. **`before_startup`**: Early-stage configuration or environment checks. +2. **`on_startup`**: Core initialization (e.g., DB connections, Cache priming). +3. **`after_startup`**: Post-init tasks (e.g., sending "System Up" notifications). +During shutdown, plugins are processed in **reverse topological order** to ensure that shared resources (like the Database) are closed only after the modules depending on them have safely shut down. -Now that our plugin is ready, we will look at how ZCore's **Kernel** coordinates these modules in the final bootstrapping step. \ No newline at end of file +### 4. Dependency Integrity Check +Before starting, ZCore verifies that every string in a plugin's `dependencies` list corresponds to an actually registered plugin [kernel/engine.py]. If a dependency is missing, ZCore aborts the startup and logs exactly which plugin is missing its requirement. \ No newline at end of file diff --git a/docs/learn/7-kernel.md b/docs/learn/7-kernel.md index 5c75c9d..9b11dbe 100644 --- a/docs/learn/7-kernel.md +++ b/docs/learn/7-kernel.md @@ -1,69 +1,101 @@ -# 🧠 Step 7: Framework Orchestration (The Kernel) +# Framework Engine & Orchestration -The **Kernel** is the modest central coordinator of every ZCore application. It acts as the "brain" that manages your plugins, resolves their dependencies, and ensures that every part of your system starts and stops in the correct sequence. +The central nervous system coordinating plugin lifecycles, dependency resolution, and concurrent application events. --- -## πŸ› οΈ The Kernel's Responsibilities - -Instead of manually importing and starting every module in your project, you hand them over to the Kernel. Here is a breakdown of what the Kernel handles for you: - -| Task | Purpose | -| :--- | :--- | -| 🧩 **Plugin Registration** | Keeping track of all active domain modules. | -| πŸ“ **Dependency Analysis** | Figuring out which plugins need to start first. | -| ⏱️ **Lifespan Management** | Triggering the `startup` and `shutdown` hooks we defined in Step 6. | -| πŸ’‰ **Core Injection** | Registering global tools (like the Event Dispatcher) into the IoC container. | +
+
+ Type
+ Core Orchestrator +
+
+ Status
+ Core Infrastructure +
+
+ Underlying Tech
+ graphlib / asyncio +
+
+ +## The Challenge +FastAPI provides a powerful `lifespan` hook, but as the number of modules grows, managing initialization order becomes a liability. Developers often resort to: +1. **Fragile Sequencing:** Manually ordering database connections, cache priming, and background workers in a single function. A single misplaced line causes runtime failures. +2. **Coupled Event Handling:** Directly calling functions across modules (e.g., `UserService` calling `EmailService`), which creates tight coupling and makes the system harder to test or modify. +3. **Startup Bottlenecks:** Executing every initialization task sequentially, unnecessarily increasing the time it takes for the application to reach a "Healthy" state. + +## The ZCore Elegance +The `Kernel` manages the application lifecycle through **Topological Orchestration**. It builds a dependency graph of your plugins and ensures each one starts and stops in the mathematically correct order. Complementing this, the `EventDispatcher` provides a loosely coupled communication bus where handlers execute concurrently without blocking the main event loop. + +=== "ZCore Kernel & Events" + :::python + # 1. Orchestration + kernel = Kernel() + kernel.add_plugin(AuthPlugin()) + kernel.add_plugin(DatabasePlugin()) # Sorted automatically + + # 2. Decoupled Communication + @kernel.dispatcher.subscribe("user_registered") + async def send_welcome_email(payload): + # Executes concurrently with other listeners + await email_provider.send(...) + + # 3. Clean Integration + app = FastAPI(lifespan=kernel.lifespan) + kernel.setup(app) + +=== "FastAPI Raw Implementation" + :::python + # Manual, fragile ordering in a single function + @asynccontextmanager + async def lifespan(app: FastAPI): + # 1. Manual Startup Order (Risk of failure if out of sync) + await init_db() + await init_auth() + + # 2. Tight Coupling (Email must be called directly) + # No built-in way to dispatch events concurrently easily + try: + yield + finally: + # 3. Manual Shutdown Order + await close_auth() + await close_db() + + app = FastAPI(lifespan=lifespan) --- -## πŸ“ Topological Dependency Sorting - -When you have a complex system, plugins often rely on one another. For example, a `PaymentPlugin` might require a `DatabasePlugin` to be ready first. +## Orchestration Workflow +The Kernel transforms your declarative plugin list into an executed lifecycle through a rigorous sorting and execution pipeline. -The Kernel automatically analyzes the `dependencies` list of every plugin and builds a **Directed Acyclic Graph (DAG)**. It then sorts them "topologically." This is a modest way of saying it creates a perfect "to-do list" where every prerequisite is finished before the next step begins. - -```mermaid -flowchart TD - DB[1. Core Database] -->|Prerequisite| Auth[2. Authentication Plugin] - Auth -->|Prerequisite| Products[3. Product Management] - Products -->|Prerequisite| Analytics[4. Usage Analytics] -``` +

+ The Unified Request Journey +

--- -## πŸ’» Initializing the Kernel - -To prepare your application, you simply create a Kernel instance and add your plugins to it. This usually happens in your `main.py` file: - -```python -from zcore import Kernel -from products.plugin import ProductPlugin +## Boundaries & Integration +The Kernel and Dispatcher are designed to wrap around FastAPI's native async capabilities without obscuring them. -# 1. Initialize the Kernel instance -kernel = Kernel() - -# 2. Register your domain plugins -kernel.add_plugin(ProductPlugin()) -``` +* **Lifespan Injection:** The `kernel.lifespan` method is a standard Python asynchronous context manager. It is passed directly to the `FastAPI` constructor, meaning the native Starlette lifecycle is what actually drives ZCore. +* **Decoupled Events:** The `EventDispatcher` is registered as a singleton in the `container`. You can inject it into any `BaseService` or `BaseRepository` using `Inject(EventDispatcher)` to trigger events from deep within your domain logic. +* **Bypass:** If you don't want to use the Plugin system, you can still use the `EventDispatcher` as a standalone library within your project to handle internal signaling. --- -## πŸ’‘ Engineering Insights - -!!! info "πŸ›‘οΈ Safety Against Cyclic Loops" - If you accidentally create a "circular dependency" (e.g., Plugin A depends on B, and B depends on A), the Kernel will detect this during startup and raise a `RuntimeError`. This prevents your application from entering an infinite loop or crashing in an unpredictable state. - -!!! tip "πŸ’‘ Decoupled Communication" - The Kernel also manages the **Event Dispatcher**. This allows plugins to talk to each other without being "hard-coded" together. For example, the `ProductPlugin` can emit an event that the `AnalyticsPlugin` listens to, even if they don't know each other exists. - ---- +## Under-the-Hood Spec -## πŸ” Troubleshooting the Kernel +### 1. Reverse-Topological Shutdown +While startup follows a dependency path (A -> B), the Kernel ensures that shutdown occurs in **reverse order** (B -> A) [kernel/engine.py]. This ensures that shared resources, like database pools, are only closed after the domain modules using them have safely terminated. -If your application fails to start, check the following common modest mistakes: +### 2. Concurrent Event Execution +The `EventDispatcher.dispatch` method schedules all registered async handlers using `asyncio.gather` [kernel/events.py]. Crucially, it sets `return_exceptions=True` and logs errors within the dispatcher. This prevents a failure in a single listener (e.g., a failing email server) from crashing the entire request or the application. -* **Missing Dependency:** You listed a dependency in `plugin.py`, but you forgot to register that plugin with `kernel.add_plugin()`. -* **Startup Exception:** One of your plugins crashed inside `on_startup()`. Because the Kernel executes these in order, a crash in one plugin will safely prevent the rest of the system from starting in an unstable state. +### 3. Missing Dependency Assertion +During the `_resolve_dependencies` phase, the Kernel performs a cross-reference check [kernel/engine.py]. If a plugin declares a dependency string that does not exist in the active registry, the Kernel raises a descriptive `RuntimeError`, preventing the application from starting in an incomplete state. -In our final step, we will assemble all these pieces in `main.py` and launch the application! \ No newline at end of file +!!! info "Event Handlers" + The Dispatcher supports both standard synchronous functions and `asyncio` coroutines. Synchronous handlers are executed sequentially, while asynchronous ones are gathered for concurrent execution. \ No newline at end of file diff --git a/docs/learn/8-main.md b/docs/learn/8-main.md index b12707a..3b6f0b2 100644 --- a/docs/learn/8-main.md +++ b/docs/learn/8-main.md @@ -1,124 +1,117 @@ -# πŸš€ Step 8: Bootstrapping the Application - -We have reached the final modest step of our bottom-up quickstart guide. In this stage, we will assemble all our componentsβ€”the database manager, the core kernel, and our domain pluginsβ€”into a single entry point: `main.py`. - -Open `main.py` and replace its contents with the following assembly: - -```python -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -from fastapi import FastAPI -from zcore import Kernel, settings -from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware -from zcore.exceptions import app_exception_handler, AppException -from zcore.db import db_manager, register_db_event_dispatcher -from zcore.logging import setup_logging - -# Import our custom domain plugin -from products.plugin import ProductPlugin - -# 1. Initialize Structured Logging -setup_logging() - -# 2. Configure our Database Manager (Async SQLite Engine) -db_manager.init_app( - db_url=settings.DATABASE_URL, - pool_size=settings.POOL_SIZE, - max_overflow=settings.MAX_OVERFLOW, - echo=(settings.ENVIRONMENT == "development") -) - -# 3. Create the Core Kernel and register plugins -kernel = Kernel() -kernel.add_plugin(ProductPlugin()) - -# 4. Register the global database event dispatcher -register_db_event_dispatcher(kernel.dispatcher) - -# 5. Initialize the FastAPI app and hook the kernel's lifespan -app = FastAPI( - title=settings.PROJECT_NAME, - lifespan=kernel.lifespan -) - -# 6. Apply core middlewares -app.add_middleware(RequestLogMiddleware) -app.add_middleware(ScopedDependencyMiddleware) - -# 7. Apply the centralized error handler -app.add_exception_handler(AppException, app_exception_handler) - -# 8. Setup plugin routes -kernel.setup(app) - -@app.get("/") -async def root(): - return { - "status": "healthy", - "framework": "ZCore", - "environment": settings.ENVIRONMENT - } -``` +# Application Bootstrapping ---- - -## πŸ—οΈ The Bootstrapping Sequence - -Understanding the order of operations helps in maintaining a predictable system. Here is what happens when you start this script: - -| Phase | Responsibility | Why it matters | -| :--- | :--- | :--- | -| πŸͺ΅ **Logging** | `setup_logging()` | Standardizes logs into JSON or Console format. | -| πŸ—„οΈ **Database** | `db_manager.init_app()` | Warms up the async connection pool. | -| 🧠 **Kernel** | `Kernel()` | Prepares the dependency graph for your plugins. | -| πŸ›‘οΈ **Security** | `Middlewares` | Manages Request-IDs and Scoped DI boundaries. | -| 🚦 **Lifespan** | `kernel.lifespan` | Orchestrates the `on_startup` and `on_shutdown` hooks. | +The unified entry point where infrastructure, security, and domain plugins converge into a running FastAPI instance. --- -## πŸ’» Launching the Server +
+
+ Type
+ Entry Point +
+
+ Status
+ Mandatory Root +
+
+ Underlying Tech
+ FastAPI / Kernel +
+
+ +## The Challenge +In many FastAPI projects, the `main.py` file eventually becomes a "God Object"β€”a massive procedural script that handles logging, database initialization, middleware ordering, exception mapping, and router registration. This leads to: +1. **Fragile Middleware Ordering:** Placing `Authentication` after `Routing` or `Session` management after `Logging`, causing cryptic errors. +2. **Infrastructure Leaks:** Database connection strings and logging configurations mixed with business route definitions. +3. **Hard-to-Test Entry Points:** Because everything is tightly coupled in one file, spinning up the app for integration tests without the entire stack becomes difficult. + +## The ZCore Elegance +ZCore provides a declarative bootstrap template. Instead of writing procedural logic, you configure the `Kernel`, register your domain `Plugins`, and attach the core middlewares. The `main.py` serves strictly as an **Orchestrator**, delegating the heavy lifting to the infrastructure managers and domain plugins. + +=== "ZCore main.py (Orchestrated)" + :::python + # Auto-generated via 'zc init' + from fastapi import FastAPI + from zcore import Kernel, settings + from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware + from zcore.exceptions import app_exception_handler, AppException + from zcore.db import db_manager, register_db_event_dispatcher + from zcore.logging import setup_logging + + # 1. Infrastructure Setup + setup_logging() + db_manager.init_app(db_url=settings.DATABASE_URL) + + # 2. Kernel & Plugin Registration + kernel = Kernel() + kernel.add_plugin(UsersPlugin()) # Your domain logic + register_db_event_dispatcher(kernel.dispatcher) + + # 3. FastAPI Bootstrap + app = FastAPI(title=settings.PROJECT_NAME, lifespan=kernel.lifespan) + kernel.setup(app) + + # 4. Middleware & Error Handlers + app.add_middleware(RequestLogMiddleware) + app.add_middleware(ScopedDependencyMiddleware) + app.add_exception_handler(AppException, app_exception_handler) + +=== "FastAPI Raw (Procedural)" + :::python + # Typical 'main.py' in a standard project + app = FastAPI() + + @app.on_event("startup") + async def startup(): + # Manual procedural setup + logging.config.dictConfig(LOG_CONFIG) + await database.connect() + # What if this fails? Is the error caught? + await cache.connect() + + # Manual Error Handling for every custom exception + @app.exception_handler(MyCustomError) + async def handler(request, exc): + return JSONResponse(...) + + # Middleware ordering is manual and invisible + app.add_middleware(SessionMiddleware) + app.add_middleware(AuthMiddleware) + + app.include_router(users_router) + # File grows to 500+ lines... -With the entry point configured, you can launch the local development server using the ZCore CLI. Ensure your virtual environment is active: +--- -```bash -zc run -``` +## Bootstrapping Lifecycle +When you launch your application (via `zc run`), ZCore executes the following phases in a strict, predictable sequence: -You should see log output indicating that Uvicorn has started on `http://127.0.0.1:8000`. +

+ Bootstrapping Lifecycle +

--- -## βœ… Quickstart Success Checklist +## Boundaries & Integration +The ZCore bootstrap process is designed to be a "wrapper" that stays out of your way. -To verify that your Product Management System is running as intended, check the following: - -* [ ] **API Health Check:** Open `http://127.0.0.1:8000/` in your browser. You should receive: `{"status": "healthy", ...}`. -* [ ] **Swagger UI:** Navigate to `http://127.0.0.1:8000/docs`. You will see the auto-generated documentation for the `/products` CRUD suite. -* [ ] **Database Generation:** Since SQLite is our default, ZCore will automatically create a `zcore_dev.db` file in your root folder upon the first request. - -```text -πŸ“ product_api/ -β”œβ”€β”€ products/ ← Your domain module -β”œβ”€β”€ main.py ← Assembly point -β”œβ”€β”€ zcore_dev.db ← Generated SQLite database -β”œβ”€β”€ .env ← Configuration -└── requirements.txt ← Project dependencies -``` +* **FastAPI Native:** The `app` variable is a standard `FastAPI()` instance. You can use any native FastAPI feature, such as custom lifespan context managers or third-party middlewares, alongside ZCore. +* **Infrastructure Isolation:** Database and Logging initialization happen before the app starts. This ensures that infrastructure is "warm" before any plugin logic is executed. +* **Manual Overrides:** If you need to add a router that isn't part of a plugin, you can simply call `app.include_router()` in `main.py` as you normally would. --- -## πŸ’‘ Engineering Insights +## Under-the-Hood Spec -!!! tip "πŸ’‘ Response Customization" - If you want to change the database or server port, you don't need to touch `main.py`. Simply update the values in your `.env` file. ZCore's `settings` proxy will automatically pick up the changes. +### 1. Scoped Middleware Dependency +The `ScopedDependencyMiddleware` is critical. It initializes a unique `uuid4` correlation ID for every request and sets up the request-scoped DI container [web/middleware.py]. Most importantly, it injects the active `AsyncSession` into the container, ensuring that all services and repositories within that request share the same database transaction. -!!! info "πŸ›‘οΈ Middleware Safety" - The `ScopedDependencyMiddleware` is critical. It ensures that every HTTP request gets its own isolated "sandbox" for database sessions and services. When the request ends, ZCore automatically cleans up these resources to prevent memory leaks. - ---- +### 2. Centralized Exception Mapping +The `app_exception_handler` intercepts any exception inheriting from `AppException` [exceptions/handlers.py]. It automatically logs the error with structured metadata (request path, method, and correlation ID) and wraps the error message in the standardized `ResponseWrapper` JSON envelope, ensuring your API never returns a raw traceback. -**Congratulations!** You have successfully built a modular, type-safe, and architecturally sound API using ZCore. +### 3. The DB-Event Bridge +By calling `register_db_event_dispatcher(kernel.dispatcher)`, you bridge the database layer with the application event system [db/events.py]. This allows low-level database operations (like a transaction commit) to trigger high-level domain events (like sending a notification) through a loosely coupled, non-blocking bus. -From here, you can add more plugins, implement complex business logic in your services, or explore the **Unit of Work** pattern for managing multiple database operations in a single transaction. \ No newline at end of file +!!! success "Project Ready" + You have now completed the core ZCore Tutorial. Your project is scaffolded, your infrastructure is secure, and your domain logic is structured. Proceed to the **How-To Guides** to implement advanced features like JWT Auth or Multi-Repository Transactions. \ No newline at end of file diff --git a/docs/learn/installation.md b/docs/learn/installation.md index 51c8ded..1ded8c2 100644 --- a/docs/learn/installation.md +++ b/docs/learn/installation.md @@ -1,94 +1,105 @@ -# πŸ“¦ Installation & Project Scaffolding +# Installation & Scaffolding -Before we write code, we need to set up our Python virtual environment, install the ZCore framework, and scaffold the initial project directories. +Standardize and scaffold your architectural foundation in seconds using the ZCore CLI. --- -## πŸ“‹ Prerequisites - -* 🐍 Python 3.11 or higher. -* πŸ“¦ `pip` (Python package installer). +
+
+ Type
+ CLI Utility +
+
+ Status
+ Core Tool +
+
+ Underlying Tech
+ Pathlib / Subprocess +
+
+ +## The Challenge +Unlike frameworks like Django, FastAPI does not provide a built-in "admin" or scaffolding tool. Developers often spend the first few hours of a project manually creating directories, setting up `main.py` entry points, configuring `.env` files, and writing repetitive boilerplate for every new domain module. This lack of structure leads to inconsistent layouts across different projects within the same team. + +## The ZCore Elegance +ZCore provides the `zc` command-line interface. It handles project initialization, environment configuration, and modular "app" scaffolding with optional boilerplate templates, ensuring every project follows the same clean architectural patterns from day one. + +=== "ZCore Scaffolding" + :::bash + # 1. Install the framework + pip install fastapi-zcore-framework[all] + + # 2. Initialize a new project + zc init my_awesome_api + cd my_awesome_api + + # 3. Create a modular app with full boilerplate + zc startapp billing --template + + # 4. Launch development server + zc run + +=== "FastAPI Manual Setup" + :::bash + # Standard FastAPI requires manual setup for everything: + mkdir my_api && cd my_api + touch main.py .env requirements.txt + # (Open editor, manually copy-paste FastAPI boilerplate) + # (Manually create subdirectories for billing) + mkdir billing + touch billing/models.py billing/schemas.py billing/routers.py + # (Write repetitive CRUD logic for the 100th time...) + uvicorn main:app --reload --- -## πŸ› οΈ Step 1: Install ZCore +## Getting Started -We suggest setting up a clean virtual environment first to isolate your dependencies: +### 1. Installation +Install ZCore using `pip`. It is recommended to use a virtual environment. The `[all]` extra includes SQLAlchemy, Redis, and Argon2 support. ```bash -# Create and activate virtual environment -python -m venv .venv -source .venv/bin/activate # On Windows, use: .venv\Scripts\activate - -# Install ZCore with all optional dependencies (SQLAlchemy, Redis, Crypto, etc.) pip install fastapi-zcore-framework[all] ``` ---- - -## πŸ—οΈ Step 2: Initialize the Project Blueprint - -ZCore comes equipped with a modest command-line utility, `zc`, to manage code scaffolding. Let's create our project shell: +### 2. Initializing a Project +The `init` command creates the project root, a pre-configured `main.py`, a `.env` file with a generated `SECRET_KEY`, and a `requirements.txt`. ```bash -zc init product_api -cd product_api +zc init my_project ``` -This command generates a structured layout: - -* πŸ“ `main.py`: The system entry point and kernel bootstrapping. -* πŸ“ `.env`: Local environment settings (pre-configured with `PYTHONPATH`, `SECRET_KEY`, and database URL). -* πŸ“ `requirements.txt`: Standard project requirements. -* πŸ“ `.gitignore`: Safe exclusion defaults for Git. +### 3. Creating a Modular App +ZCore promotes a modular domain-driven structure. Each "app" is a directory containing its own models, services, and routers, wrapped in a `Plugin` class. -The generated `.env` file comes pre-configured for development: - -```env -PYTHONPATH=. -ENVIRONMENT=development -PROJECT_NAME=product_api -DATABASE_URL=sqlite+aiosqlite:///zcore_dev.db -SECRET_KEY= -``` +* **Clean App:** Creates empty files for a clean slate. + ```bash + zc startapp orders + ``` +* **Template App:** Populates files with ZCore's standard CRUD/Repository boilerplate. + ```bash + zc startapp orders --template + ``` --- -## πŸš€ Step 3: Scaffold the Products Domain Module +## Boundaries & Integration +The `zc` tool is a convenience utility, not a requirement. -Now we will generate the boilerplate code for our modular product system. Run the following command: - -```bash -zc startapp products -t -``` - -The `-t` flag instructs the CLI to populate the files with scaffolding templates. - -| Generated File | Responsibility | -| :--- | :--- | -| πŸ“ `products/__init__.py` | Package namespace initialization. | -| πŸ“ `products/models.py` | SQLAlchemy database model definition. | -| πŸ“ `products/schemas.py` | Pydantic data contract validation rules. | -| πŸ“ `products/repositories.py` | Data access wrapper matching Pydantic to SQL. | -| πŸ“ `products/services.py` | Domain service layer to orchestrate business transactions. | -| πŸ“ `products/routers.py` | Web layer mapping HTTP routes to services. | -| πŸ“ `products/plugin.py` | Kernel wrapper that registers the app module. | +* **Standard Python:** All files generated are standard `.py` files. You can modify, move, or delete them without breaking the CLI. +* **Uvicorn Integration:** The `zc run` command is a wrapper for `uvicorn main:app --reload`. It automatically reads `HOST` and `PORT` from your `.env` file and ensures your `PYTHONPATH` is set correctly. +* **Manual Control:** You are never locked into the CLI. If you prefer to write your own project structure from scratch, you can still import and use ZCore components (like `Kernel`, `BaseRouter`, or `BaseService`) manually. --- -## πŸ’» CLI Reference Guide - -For quick reference, here are the most common commands available in the ZCore CLI tool: +## Under-the-Hood Spec -| Command | Usage | Description | -| :--- | :--- | :--- | -| `zc init ` | `zc init core_api` | Creates a new core project directory. | -| `zc startapp ` | `zc startapp orders` | Creates an empty application module. | -| `zc startapp -t` | `zc startapp payments -t` | Scaffolds a module with boilerplates. | -| `zc run` | `zc run` | Launches the local Uvicorn reload server. | -| `zc gensecret` | `zc gensecret` | Generates a secure, 64-character secret key. | +### 1. Cryptographically Secure Scaffolding +When you run `zc init`, the CLI uses Python's `secrets` module to generate a 64-character hex string for your `SECRET_KEY`. This ensures that even development environments start with high-entropy secrets, preventing common security oversights. ---- +### 2. Environment-Aware Execution +The `zc run` command performs a sanity check to ensure `main.py` exists in the current directory. It dynamically injects the current working directory into `sys.path` (via `PYTHONPATH`) before launching Uvicorn, allowing modular imports to work seamlessly without manual path hacking. -!!! tip "πŸ’‘ Development Mode" - Keep your terminal open. In the next steps, we will write our code inside the generated `products/` folder to build out our Product Management System. \ No newline at end of file +### 3. Smart Path Evaluation +The `startapp` command uses `pathlib` to resolve absolute paths. It verifies that the requested app name is a valid Python identifier and checks for directory collisions before writing any files, preventing accidental data loss or malformed module names. \ No newline at end of file diff --git a/docs/learn/overview.md b/docs/learn/overview.md index 98223fb..41b9f02 100644 --- a/docs/learn/overview.md +++ b/docs/learn/overview.md @@ -1,39 +1,100 @@ -# πŸ—ΊοΈ Quickstart Overview: The Blueprint +# Overview -Welcome to the ZCore Quickstart guide! This tutorial is designed to take you from a clean slate to a fully functional, structured, and maintainable REST API using ZCore's domain-driven approach. - -To help you learn the framework effectively, we will build a continuous, real-world project: **The Product Management System**. This system will handle creating, retrieving, updating, searching, and deleting products with full transaction management, data validation, and safety controls. +Standardize your development without sacrificing the freedom of FastAPI. ZCore provides the architectural "chassis" for your application, coordinating dependencies, security context, and database transactions into a unified, predictable lifecycle. --- -## πŸ—οΈ Why a "Bottom-Up" Engineering Approach? +
+
+ Type
+ Architectural Layer +
+
+ Status
+ Recommended Core +
+
+ Underlying Tech
+ FastAPI / Starlette +
+
+ +## The Challenge +While FastAPI is exceptional for building high-performance APIs, it provides no structural blueprint. As projects scale, developers often encounter: +- **Global Dependency Clutter:** Difficulty managing different lifecycles (Singleton vs. Scoped) across modules. +- **Transactional Inconsistency:** Scattered database commits leading to partial data writes. +- **Security Boilerplate:** Manual field masking and permission checks repeated across dozens of routes. +- **Plugin Fragmentation:** Hard-to-track initialization orders for background workers, caches, and database engines. + +## The ZCore Elegance +ZCore solves this by introducing a **Plugin-based Kernel**. Instead of manually wiring everything in a bloated `main.py`, you register domain-specific plugins. ZCore calculates the correct initialization order and manages the application lifespan automatically. + +=== "`ZCore Plugin Orchestration`" + + :::python + # Define a clean, isolated plugin for your domain + class PaymentPlugin(Plugin): + name = "payments" + dependencies = ["database", "auth"] # ZCore sorts these automatically + + def setup(self, app: FastAPI): + app.include_router(payment_router) + + async def on_startup(self): + await payment_gateway.connect() + + # In your main.py: Simple, declarative registration + kernel = Kernel() + kernel.add_plugin(PaymentPlugin()) + app = FastAPI(lifespan=kernel.lifespan) + kernel.setup(app) -ZCore's suggested design approach is **Bottom-Up**. We start from the physical database representation and move upward to the presentation layer. By establishing a solid foundation at the data layer, we ensure that schemas, repositories, and services remain tightly bound, type-safe, and highly predictable. +=== "`Standard FastAPI Manual Setup`" -```mermaid -flowchart TD - DB[(Database SQL)] -->|Mapped By| Model[1. Database Model] - Model -->|Validated By| Schemas[2. Pydantic Schemas] - Schemas -->|Queried By| Repo[3. Repository Layer] - Repo -->|Orchestrated By| Service[4. Business Service] - Service -->|Exposed By| Router[5. API Router] - Router -->|Packaged By| Plugin[6. Plugin Wrapper] - Plugin -->|Sorted By| Kernel[7. Core Kernel] - Kernel -->|Bootstrapped By| Main[8. FastAPI Application] -``` + :::python + # Standard FastAPI requires manual, sequential ordering in main.py + app = FastAPI() + + @app.on_event("startup") + async def startup(): + # Manual ordering: If database isn't ready, payment fails. + # This becomes a "main.py" file with 500+ lines in large projects. + await init_db() + await init_auth() + await payment_gateway.connect() + + app.include_router(payment_router) + app.include_router(auth_router) + # ... and 20 other routers --- -## πŸ“Š Scenario Parameters +## The Unified Request Journey +ZCore coordinates the request lifecycle through specialized middlewares and context managers, ensuring that every coroutine knows the current user and their data restrictions. + +

+ The Unified Request Journey +

+ +--- -Our `Product` domain will manage the following metrics: +## Boundaries & Integration +ZCore is designed as a **complementary chassis**. It does not hide FastAPI; it stabilizes it. -* πŸ†” **ID:** Unique global identifier (UUIDv4). -* πŸ“ **Name:** The descriptive name of the item. -* πŸ’° **Price:** The retail value of the product (which must never be negative). -* πŸ“¦ **Stock:** The integer quantity of items physically available in inventory. +* **Pydantic Compatibility:** `Zchema` inherits directly from Pydantic's `BaseModel`. You can use standard Pydantic features (validators, computed fields) exactly as you always have. +* **Starlette Middlewares:** ZCore middlewares are standard ASGI/Starlette implementations. You can mix them with any other third-party FastAPI middleware. +* **Bypass Strategy:** If you have a route that doesn't fit the `BaseRouter` pattern, you can write a standard FastAPI `@app.get("/")` path function. You can still use `Inject()` to retrieve services or repositories from the ZCore container within that function. --- -!!! info "πŸ’‘ The Philosophy of Clean Abstractions" - FastAPI provides the asynchronous routing engine; ZCore layers structural safety on top of it. In the following sections, we will walk through how each layer encapsulates a single responsibility, reducing boilerplate code and making your codebase easier to maintain. \ No newline at end of file +## Under-the-Hood Spec + +### 1. Topological Dependency Resolution +The Kernel uses `graphlib.TopologicalSorter` to analyze the `dependencies` attribute of every registered plugin. This builds a Directed Acyclic Graph (DAG), ensuring that if Plugin B depends on Plugin A, Plugin A's `on_startup` always completes before Plugin B begins. + +### 2. High-Performance Reflection Cache +The Dependency Injection (DI) container utilizes a signature cache. During the first resolution of a class, ZCore uses the `inspect` module to analyze constructor type hints. This metadata is cached, allowing subsequent resolutions to skip reflection overhead entirely. + +### 3. Contextual Isolation +ZCore utilizes Python's `contextvars` to maintain thread-safe and coroutine-safe state. The `request_context` manager ensures that `current_user_id` and `restricted_fields` are never leaked between concurrent requests, even in a high-concurrency async environment. \ No newline at end of file diff --git a/docs/learn/quickstart.md b/docs/learn/quickstart.md new file mode 100644 index 0000000..6d623a2 --- /dev/null +++ b/docs/learn/quickstart.md @@ -0,0 +1,431 @@ +# 10-Minute Quickstart: Build a Task Manager API + +Build a production-grade Task Manager API using ZCore in under 10 minutes. This guide covers the entire journey β€” from an empty directory to a running server with secure, scaffolded CRUD endpoints, dynamic field masking, and database persistence. + +--- + +
+
+ Type
+ Hands-On Tutorial +
+
+ Duration
+ ~10 Minutes +
+
+ Difficulty
+ Beginner +
+
+ +--- + +## What We're Building + +A **Task Manager API** that supports: + +- βœ… Automated creation, retrieval, updates, and deletion of tasks +- βœ… Built-in keyset (cursor) and offset pagination +- βœ… Advanced relational and conditional searches out-of-the-box +- βœ… Context-aware data masking to protect sensitive fields (like assignee emails) +- βœ… Scalable execution layers with structured lifecycle hooks + +By the end, you'll have **7 fully functional endpoints** with namespaced authorization actions, running atop ZCore's Scoped Dependency Injection and Plugin lifecycle architecture. + +--- + +## Step 1: Install & Initialize + +Create a clean virtual environment and install the framework along with its database and cryptographic extras: + +```bash +# Create and activate a virtual environment +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install ZCore with all optional dependencies +pip install fastapi-zcore-framework[all] + +# Initialize a new project +zc init taskmanager +cd taskmanager +``` + +The `zc init` command generates the following clean root structure: + +``` +taskmanager/ +β”œβ”€β”€ main.py # Application entry point & orchestration +β”œβ”€β”€ .env # Project environment settings (including secure SECRET_KEY) +β”œβ”€β”€ requirements.txt # Dependencies list +β”œβ”€β”€ .gitignore # Python, IDE, and DB exclusions +└── zcore_dev.db # Local development SQLite database +``` + +--- + +## Step 2: Scaffold the Tasks Domain Module + +ZCore enforces a **modular, domain-driven architecture**. Each feature context lives in its own directory, grouping its data model, projection schemas, persistence unit, business logic, and routing orchestrator into a single package. + +Generate the `tasks` domain module using the built-in scaffolding tool: + +```bash +zc startapp tasks --template +``` + +This commands scaffolds a clean, layered structure inside the `tasks` directory: + +``` +tasks/ +β”œβ”€β”€ __init__.py +β”œβ”€β”€ models.py # SQLAlchemy Declarative Model +β”œβ”€β”€ schemas.py # Context-Aware Zchema Definitions +β”œβ”€β”€ repositories.py # Persistence & Query Layer +β”œβ”€β”€ services.py # Pure Business Logic & Lifecycle Hooks +β”œβ”€β”€ routers.py # Scaffolded CRUD Router +└── plugin.py # Kernel Lifespan Plugin Wrapper +``` + +Let's modify each component to build our task manager. + +--- + +## Step 3: Define the Database Model + +Open `tasks/models.py` and configure the database table schema. We will use modern SQLAlchemy 2.0 type mapping: + +```python +import uuid +from sqlalchemy import String, Boolean +from sqlalchemy.orm import Mapped, mapped_column +from zcore import Base + +class Task(Base): + __tablename__ = "tasks" + + id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4) + title: Mapped[str] = mapped_column(String(255), index=True) + description: Mapped[str | None] = mapped_column(String(1024), default=None) + priority: Mapped[str] = mapped_column(String(50), default="medium") + is_completed: Mapped[bool] = mapped_column(Boolean, default=False) + assignee_email: Mapped[str | None] = mapped_column(String(255), default=None) +``` + +!!! info "Auto-Generated Permission Scopes" + By inheriting from `Base`, ZCore automatically exposes a descriptive `Actions` descriptor on the model based on its `__tablename__` [setup.py]:
+ - `Task.actions().CREATE` → `"tasks:create"`
+ - `Task.actions().VIEW` → `"tasks:view"`
+ - `Task.actions().UPDATE` → `"tasks:update"`
+ - `Task.actions().DELETE` → `"tasks:delete"`
+ - `Task.actions().LISTVIEW` → `"tasks:listview"`
+ +--- + +### Step 4: Define Context-Aware Schemas + +Open `tasks/schemas.py`. We will implement `Zchema` instead of Pydantic's raw `BaseModel` to handle data-masking at the serialization boundary: + +```python +from pydantic import ConfigDict +import uuid +from zcore import Zchema + +class TaskBase(Zchema): + __model__ = "tasks" # Binds schema to the "tasks" permission namespace + +class TaskCreate(TaskBase): + title: str + description: str | None = None + priority: str = "medium" + assignee_email: str | None = None + +class TaskUpdate(TaskBase): + title: str | None = None + description: str | None = None + priority: str | None = None + is_completed: bool | None = None + assignee_email: str | None = None + +class TaskResponse(TaskBase): + id: uuid.UUID + is_completed: bool + + model_config = ConfigDict(from_attributes=True) +``` + +!!! info "Unified Schema Projection & Mass Assignment Protection" + In traditional FastAPI, developers suffer from "Schema Explosion"β€”creating separate classes like `TaskCreate`, `TaskUpdate`, `TaskPublicResponse`, and `TaskAdminResponse` just to handle field-level permissions. + + With ZCore's `Zchema`, you can dramatically collapse this hierarchy [projection.py]. By defining a single, unified schema with all potential attributes, ZCore's `filter_restricted_inputs` and `secure_serializer` hooks automatically intercept requests [projection.py]. If `"tasks.assignee_email"` is added to the active `restricted_fields` context: + 1. **Input Shielding:** If a malicious user tries to POST or PUT `"assignee_email"`, ZCore silently prunes it *before* validation, preventing **Mass Assignment** exploits [projection.py]. + 2. **Output Masking:** When returning data, ZCore automatically strips `"assignee_email"` from the JSON response before serialization, preventing **Data Leakage** [projection.py]. + + This allows you to manage security dynamically at runtime using a single source of truth, rather than maintaining dozens of redundant Pydantic models. +--- + +## Step 5: Configure the Repository Layer + +Open `tasks/repositories.py`. The repository abstracts raw database operations using the asynchronous SQLAlchemy engine: + +```python +from zcore import BaseRepository +from sqlalchemy.ext.asyncio import AsyncSession +from .models import Task +from .schemas import TaskCreate, TaskUpdate + +class TaskRepository(BaseRepository[Task, TaskCreate, TaskUpdate]): + def __init__(self, db: AsyncSession): + super().__init__(model=Task, db=db) +``` + +No query code is required. `BaseRepository` automatically provides high-performance implementations of `get`, `get_by_ids` (with instant fast-path validation), `get_list` (supporting offset & keyset cursors), `create`, `update`, `delete`, and `search` [repository.py]. + +--- + +## Step 6: Implement the Business Service + +Open `tasks/services.py` to write your core business logic and lifecycle hooks: + +```python +from zcore import BaseService, Inject +from .models import Task +from .schemas import TaskCreate, TaskUpdate +from .repositories import TaskRepository + +class TaskService(BaseService[Task, TaskCreate, TaskUpdate]): + def __init__(self, repository: TaskRepository = Inject(TaskRepository)): + super().__init__(model=Task, repository=repository) + + # --- Lifecycle Hooks --- + + async def pre_create(self, schema: TaskCreate) -> None: + """Executed automatically inside the write pipeline before DB insertion.""" + if schema.priority not in ["low", "medium", "high"]: + schema.priority = "medium" + + async def post_create(self, model: Task) -> None: + """Executed after successful DB insertion, prior to transaction commit.""" + # Clean boundary for publishing events, audit logs, or notifications + pass +``` + +The `Inject(TaskRepository)` annotation instructs ZCore's **Scoped DI Container** to resolve the repository automatically within the active request context [di.py]. + +--- + +## Step 7: Scaffold the Automated Router + +Open `tasks/routers.py`. We will extend `BaseRouter` to generate 7 secure CRUD and search endpoints: + +```python +from zcore import BaseRouter +from .schemas import TaskCreate, TaskUpdate, TaskResponse +from .services import TaskService +from .models import Task + +class TaskRouter(BaseRouter[TaskCreate, TaskUpdate]): + model = Task + create_schema = TaskCreate + update_schema = TaskUpdate + schema_out = TaskResponse + service = TaskService + + prefix = "/tasks" + tags = ["Tasks"] + expose_schemas = True # Activates runtime schema introspection + +# Instantiate for plugin integration +router_instance = TaskRouter() +``` + +By default, the router uses `ZCoreAPIRoute` to ensure responses are serialized using our custom high-performance encoder, automatically appending the `Vary` HTTP header to prevent caching leaks when field masking is active [api_router.py]. + +--- + +## Step 8: Package as a Kernel Plugin + +Open `tasks/plugin.py`. This class packages the `tasks` domain and defines its startup and shutdown dependencies: + +```python +from fastapi import FastAPI +from zcore import Plugin +from .routers import router_instance + +class TaskPlugin(Plugin): + name = "tasks" + version = "0.1.0" + dependencies = [] # Declare other plugin names that must boot up first + + def setup(self, app: FastAPI) -> None: + # Mount the scaffolded APIRouter instance to the FastAPI app + app.include_router(router_instance.router) + + async def on_startup(self) -> None: + """Standard startup lifecycles (e.g. warming up caches)""" + pass + + async def on_shutdown(self) -> None: + """Graceful cleanup tasks during application teardown""" + pass +``` + +--- + +## Step 9: Wire the App in `main.py` + +Open your root `main.py` file. This acts as the declarative orchestrator of the entire stack: + +```python +from fastapi import FastAPI +from zcore import Kernel, settings +from zcore.web import RequestLogMiddleware, ScopedDependencyMiddleware +from zcore.exceptions import app_exception_handler, AppException +from zcore.db import db_manager, register_db_event_dispatcher +from zcore.logging import setup_logging + +from tasks.plugin import TaskPlugin + +# 1. Initialize structured logging configuration +setup_logging() + +# 2. Configure the database connection pool (SQLite development fallback) +db_manager.init_app( + db_url=settings.DATABASE_URL, + echo=(settings.ENVIRONMENT == "development") +) + +# 3. Create the Kernel and register domain plugins +kernel = Kernel() +kernel.add_plugin(TaskPlugin()) + +# 4. Bridge database events to the central kernel dispatcher +register_db_event_dispatcher(kernel.dispatcher) + +# 5. Create FastAPI app bound to ZCore's topological lifespan +app = FastAPI( + title=settings.PROJECT_NAME, + lifespan=kernel.lifespan +) + +# 6. Finalize kernel plugin setups +kernel.setup(app) + +# 7. Add core architectural middlewares & exception handlers +app.add_middleware(RequestLogMiddleware) +app.add_middleware(ScopedDependencyMiddleware) +app.add_exception_handler(AppException, app_exception_handler) +``` + +--- + +## Step 10: Run and Verify + +Launch the development server with automated reload tracking: + +```bash +zc run +``` + +``` +πŸ“‘ Starting ZCore Dev Server on 127.0.0.1:8000... +INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) +INFO: Application startup complete. +``` + +Open a new terminal session and test the scaffolded endpoints: + +=== "1. Create a Task" + + :::bash + curl -X POST http://127.0.0.1:8000/tasks/ \ + -H "Content-Type: application/json" \ + -d '{ + "title": "Evaluate ZCore Framework", + "description": "Thoroughly test repository patterns and field masking", + "priority": "high", + "assignee_email": "architect@zcore.io" + }' + + ========= Structured Response Envelope: ========= + { + "success": true, + "message": "Success", + "data": { + "id": "e8a93cb4-5fa7-4f67-a068-d06e2ea851c2", + "title": "Evaluate ZCore Framework", + "description": "Thoroughly test repository patterns and field masking", + "priority": "high", + "assignee_email": "architect@zcore.io", + "is_completed": false + }, + "meta": null + } + +=== "2. Paginated Query (GET)" + + :::bash + curl http://127.0.0.1:8000/tasks/ + + ========= Response ========= + { + "success": true, + "message": "Success", + "data": [ + { + "id": "e8a93cb4-5fa7-4f67-a068-d06e2ea851c2", + "title": "Evaluate ZCore Framework", + "description": "Thoroughly test repository patterns and field masking", + "priority": "high", + "assignee_email": "architect@zcore.io", + "is_completed": false + } + ], + "meta": null + } + +=== "3. Secure Schema Extraction" + + To generate dynamic forms on your frontend without exposing restricted structures, fetch the OpenAPI JSON Schema directly from the endpoint via `?schema=true`: + + :::bash + curl http://127.0.0.1:8000/tasks/?schema=true + + ========= Context-Aware Schema Response: ========= + { + "success": true, + "message": "Schema generated successfully", + "data": { + "title": "TaskResponse", + "type": "object", + "properties": { + "title": { "type": "string" }, + "description": { "type": "string" }, + "priority": { "type": "string", "default": "medium" }, + "assignee_email": { "type": "string" }, + "id": { "type": "string", "format": "uuid" }, + "is_completed": { "type": "boolean" } + }, + "required": ["title", "id", "is_completed"] + }, + "meta": { + "restricted_fields": [] + } + } + +--- + +## Comparison Summary + +| Built-In Feature Layer | ZCore Code Volume | Standard FastAPI Equivalent | +| :--- | :--- | :--- | +| **Declarative Model** | ~12 lines (`models.py`) | ~12 lines | +| **Masking Schemas** | ~20 lines (`schemas.py`) | ~70 lines (Separate models per-role) | +| **Persistence Repos** | ~5 lines (`repositories.py`) | ~90 lines (Manual connection and queries) | +| **Orchestrated Service**| ~15 lines (`services.py`) | ~60 lines (Manual context and commits) | +| **CRUD Router** | ~12 lines (`routers.py`) | ~120 lines (Routing, auth, and pagination) | +| **Plugin Registry** | ~12 lines (`plugin.py`) | N/A (Main file spaghetti) | +| **Total Footprint** | **~76 lines** | **~350+ lines** | + +ZCore not only saves hundreds of lines of repetitive code, but it also natively enforces **thread-safe scoping**, **automated permission mappings**, **transactional safety**, and **real-time field masking** across your entire ecosystem. \ No newline at end of file diff --git a/docs/overrides.css b/docs/overrides.css new file mode 100644 index 0000000..52ed655 --- /dev/null +++ b/docs/overrides.css @@ -0,0 +1,33 @@ +[data-slot="sidebar-menu-button"] { + width: 100%; + max-width: none; + padding-top: calc(var(--spacing) * 3);; + padding-bottom: calc(var(--spacing) * 3);; +} + +[data-slot="sidebar-content"] { + width: 100%; +} + +#page-header .flex.items-center.gap-2.pt-1\.5 > button:first-child { + display: none; +} + +a[href="https://fastapi-zcore-framework.github.io/zcore/"] span.size-8 svg { + display: none; +} + +a[href="https://fastapi-zcore-framework.github.io/zcore/"] span.size-8 { + background-image: url("https://raw.githubusercontent.com/fastapi-zcore-framework/zcore/master/docs/assets/favico.png"); + background-size: contain; + background-repeat: no-repeat; + background-position: center; +} + +button[title="Toggle theme"] { + display: none !important; +} + +button[title="Toggle layout"] { + display: none !important; +} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 1bfc899..a4603bf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,17 +7,10 @@ repo_name: fastapi-zcore-framework/zcore repo_url: https://github.com/fastapi-zcore-framework/zcore theme: - name: material + name: shadcn logo: assets/logo.png favicon: assets/favico.png palette: - - media: "(prefers-color-scheme: light)" - scheme: default - primary: teal - accent: purple - toggle: - icon: material/weather-sunny - name: Switch to dark mode - media: "(prefers-color-scheme: dark)" scheme: slate primary: teal @@ -34,42 +27,31 @@ theme: plugins: - search - - mkdocstrings: - handlers: - python: - paths: [src] markdown_extensions: + - codehilite + - pymdownx.tabbed - admonition - - pymdownx.details - - pymdownx.highlight: - anchor_linenums: true - line_spans: __span - pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.snippets - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.tabbed: - alternate_style: true + +extra_css: + - overrides.css nav: - Home: index.md - - Learn (Tutorial): - - Overview: learn/overview.md - - Installation & Scaffolding: learn/installation.md - - "Step 1: Database Models": learn/1-db.md - - "Step 2: Data Schemas": learn/2-schemas.md - - "Step 3: Repository Layer": learn/3-repository.md - - "Step 4: Business Logic": learn/4-service.md - - "Step 5: Automated Routing": learn/5-router.md - - "Step 6: Domain Packaging": learn/6-plugin.md - - "Step 7: Framework Orchestration": learn/7-kernel.md - - "Step 8: Application Bootstrapping": learn/8-main.md + - Learn: + - "10-Minute Quickstart": learn/quickstart.md + - Tutorial: + - Overview: learn/overview.md + - Installation & Scaffolding: learn/installation.md + - "Step 1: Database Models": learn/1-db.md + - "Step 2: Data Schemas": learn/2-schemas.md + - "Step 3: Repository Layer": learn/3-repository.md + - "Step 4: Business Logic": learn/4-service.md + - "Step 5: Automated Routing": learn/5-router.md + - "Step 6: Domain Packaging": learn/6-plugin.md + - "Step 7: Framework Orchestration": learn/7-kernel.md + - "Step 8: Application Bootstrapping": learn/8-main.md - How-To Guides: - JWT Authentication & Scopes: how-to/auth-setup.md @@ -81,31 +63,32 @@ nav: - Architecture & Reference: - Core & Infrastructure: - - Dependency Injection: features/infrastructure/di.md - - Framework Engine: features/infrastructure/kernel.md - - Kernel Events: features/infrastructure/events.md - - ASGI Middlewares: features/infrastructure/middleware.md - - Application Entry: features/infrastructure/main.md - - Configuration Settings: features/infrastructure/config.md - - CLI Scaffolding Tool: features/infrastructure/cli.md - - System Utilities: features/infrastructure/utils.md + - Dependency Injection: features/infrastructure/di.md # Covers: IoCContainer, Scoped DI context, reflection cache, nested dependencies, Annotated, get_db + - Framework Engine: features/infrastructure/kernel.md # Covers: Plugin protocol, Central lifespan, dependency sort, missing dependency check + - Kernel Events: features/infrastructure/events.md # Covers: EventDispatcher, concurrent execution, try/except isolation + - ASGI Middlewares: features/infrastructure/middleware.md # Covers: RequestLogMiddleware, ScopedDependencyMiddleware, request correlation id + - Application Entry: features/infrastructure/main.md # Covers: main.py template, setup bootstrap + - Configuration Settings: features/infrastructure/config.md # Covers: Pydantic Settings, SettingsProxy, Lazy imports + - CLI Scaffolding Tool: features/infrastructure/cli.md # Covers: zc run, gensecret, startapp templating, path evaluation + - System Utilities: features/infrastructure/utils.md # Covers: slugify, json encoding fallback, validate_json_schema - Data & Transactions: - - Database Manager: features/database/db.md - - Repository Pattern: features/database/repository.md - - Business Services: features/database/service.md - - Unit of Work: features/database/uow.md - - Database Events: features/database/db-events.md - - Pagination Engine: features/database/pagination.md - - Dynamic Search Engine: features/database/search.md + - Database Manager: features/database/db.md # Covers: DatabaseManager, AsyncEngine, Connection pools, SQLite fallback + - Repository Pattern: features/database/repository.md # Covers: BaseRepository, Read/Write/Search Mixins, exist optimization, get_by_ids fast path + - Business Services: features/database/service.md # Covers: BaseService, safe commit context checks, post-read/write lifecycle hooks + - Unit of Work: features/database/uow.md # Covers: UnitOfWork transaction isolation, post-commit event queues + - Database Events: features/database/db-events.md # Covers: dispatch_db_event, error-isolated event propagation + - Pagination Engine: features/database/pagination.md # Covers: Cursor/Offset pagination, Cursor Keyset sorting, Base64 encoding, order_by(None) count + - Dynamic Search Engine: features/database/search.md # Covers: SearchEngine, JSON query builder, depth and max filters security, custom handlers, escaping wildcards - Security & Shielding: - - Thread-Safe Context: features/security/context.md - - Cryptographic Protocols: features/security/security.md - - API Routing Mechanics: features/security/api-router.md - - Zchema Security & Schema Pruning: features/security/zchema.md - - Centralized Exception Handler: features/security/exceptions.md + - Thread-Safe Context: features/security/context.md # Covers: contextvars, user and restricted fields, token reset/restoration safety + - Cryptographic Protocols: features/security/security.md # Covers: Argon2 parameters, symmetric/asymmetric JWT signing, token expiry + - API Routing Mechanics: features/security/api-router.md # Covers: ZCoreAPIRoute, response class preservation, vary header append logic, schema extraction + - Zchema Security & Schema Pruning: features/security/zchema.md # Covers: Zchema model_validator/serializer, Context-aware data masking, Schema pruning + - Centralized Exception Handler: features/security/exceptions.md # Covers: AppException hierarchy, app_exception_handler, standard API error envelope - Advanced Ecosystem: - - Distributed Caching: features/ecosystem/cache.md - - File Storage & Security: features/ecosystem/storage.md - - WebSocket Streaming: features/ecosystem/stream.md - - - Pocket Reference & Cheat Sheet: cheatsheet.md + - Distributed Caching: features/ecosystem/cache.md # Covers: BaseCache/TTLLRUCache fallback, Pydantic type validation on cache hits, eviction loop safety, weakref + - File Storage & Security: features/ecosystem/storage.md # Covers: LocalStorageProvider, MaxFileSize/SafeMimeType validators, Magic bytes validation, UUID truncation + - WebSocket Streaming: features/ecosystem/stream.md # Covers: StreamManager, Redis PubSub fallback, queue overflow protection, subscription context manager + + - Quick Views: + - Pocket Reference & Cheat Sheet: cheatsheet.md \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e37d832..3b1bdf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ crypto = [ ] docs = [ "mkdocs>=1.5.0", + "mkdocs-shadcn", "mkdocs-material>=9.5.0", "mkdocstrings[python]>=0.24.0", "mkdocs-gen-files>=0.5.0", diff --git a/src/zcore/__init__.py b/src/zcore/__init__.py index ad8c7a0..40d5765 100644 --- a/src/zcore/__init__.py +++ b/src/zcore/__init__.py @@ -3,6 +3,7 @@ if TYPE_CHECKING: from zcore.kernel.engine import Kernel + from zcore.kernel.plugins import Plugin from zcore.kernel.di import container, Inject from zcore.db.setup import Base, SessionDep, db_manager, get_db from zcore.db.repository import BaseRepository @@ -31,7 +32,8 @@ 'RouteKey', 'ResponseWrapper', 'ZCoreAPIRoute', - 'Zchema' + 'Zchema', + "Plugin" ] @@ -81,5 +83,11 @@ def __getattr__(name: str) -> Any: if name == "Zchema": from zcore.web.projection import Zchema return Zchema + if name == "RouteKey": + from zcore.web.base_router import RouteKey + return RouteKey + if name == "Plugin": + from zcore.kernel.plugins import Plugin + return Plugin raise AttributeError(f"module {__name__!r} has no attribute {name!r}") \ No newline at end of file diff --git a/src/zcore/cli/commands.py b/src/zcore/cli/commands.py index 9e92f4c..6c351e5 100644 --- a/src/zcore/cli/commands.py +++ b/src/zcore/cli/commands.py @@ -87,7 +87,7 @@ def start_app(app_name: str, with_template: bool = False) -> None: create_file(app_dir / filename, content) print(f"\nπŸŽ‰ Modular App '{app_name}' created successfully with its core Plugin wrapper!") - print(f"πŸ‘‰ REGISTER this plugin in your main.py:") + print("πŸ‘‰ REGISTER this plugin in your main.py:") print(f" from {app_name}.plugin import {model_name}Plugin") print(f" kernel.add_plugin({model_name}Plugin())") diff --git a/src/zcore/cli/templates.py b/src/zcore/cli/templates.py index 0a4cf7d..c7c418e 100644 --- a/src/zcore/cli/templates.py +++ b/src/zcore/cli/templates.py @@ -129,7 +129,7 @@ class {ModelName}(Base): import uuid class {ModelName}Base(Zchema): - __db_name__ = "{table_name}" + __model__ = "{table_name}" # TODO: Add your shared model attributes pass @@ -167,8 +167,8 @@ def __init__(self, repository: {ModelName}Repository = Inject({ModelName}Reposit """ ROUTER_TEMPLATE = """from typing import Any -from zcore import BaseRouter -from zcore.web import RouteKey +from zcore import BaseRouter, RouteKey + from .schemas import {ModelName}Create, {ModelName}Update, {ModelName}Response from .services import {ModelName}Service from .models import {ModelName} @@ -202,7 +202,7 @@ def get_route_dependencies(self, route_key: RouteKey, action: str) -> list[Any]: """ PLUGIN_TEMPLATE = """from fastapi import FastAPI -from zcore.kernel import Plugin +from zcore import Plugin class {ModelName}Plugin(Plugin): name = "{app_name}" diff --git a/src/zcore/db/pagination.py b/src/zcore/db/pagination.py index 9cae3b5..75d36cd 100644 --- a/src/zcore/db/pagination.py +++ b/src/zcore/db/pagination.py @@ -10,7 +10,7 @@ import math import base64 from typing import Any, Sequence, TypeVar, Generic, Optional, Type, Literal -from datetime import datetime +from datetime import datetime, timezone from pydantic import BaseModel, Field from sqlalchemy.sql.sqltypes import DateTime @@ -224,6 +224,10 @@ def _encode_cursor(self, last_item: Any) -> str: """ value = getattr(last_item, self.cursor_field, None) if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + else: + value = value.astimezone(timezone.utc) value = value.isoformat() payload = { diff --git a/src/zcore/db/repository.py b/src/zcore/db/repository.py index ecd75f9..ccd06ff 100644 --- a/src/zcore/db/repository.py +++ b/src/zcore/db/repository.py @@ -21,7 +21,7 @@ ) if TYPE_CHECKING: - from zcore.db.search import SearchRequest, SearchEngine + from zcore.db.search import SearchRequest ModelType = TypeVar("ModelType", bound=Base) CreateSchemaType = TypeVar("CreateSchemaType", bound=BaseModel) diff --git a/src/zcore/db/search.py b/src/zcore/db/search.py index a5005a5..f514b45 100644 --- a/src/zcore/db/search.py +++ b/src/zcore/db/search.py @@ -127,9 +127,9 @@ def _is_path_restricted(self, path: str, restricted_set: Set[str]) -> bool: if not restricted_set: return False - normalized_path = path.replace("resource.", "").lower() + normalized_path = path.lower() for restricted in restricted_set: - normalized_restricted = restricted.replace("resource.", "").lower() + normalized_restricted = restricted.lower() if normalized_path == normalized_restricted: return True @@ -351,7 +351,7 @@ def _coerce_value(self, col: Any, value: Any) -> Any: return date.fromisoformat(value.split("T")[0]) if python_type is datetime and isinstance(value, str): - return datetime.fromisoformat(value.replace("Z", "+00:00")) + return datetime.fromisoformat(value) if python_type is uuid.UUID and isinstance(value, str): return uuid.UUID(value) @@ -384,12 +384,18 @@ def _compare_column(self, col: Any, op: str, value: Any) -> Any: coerced_value = self._coerce_value(col, value) - if op == "eq": return col == coerced_value - if op == "ne": return col != coerced_value - if op == "gt": return col > coerced_value - if op == "lt": return col < coerced_value - if op == "ge": return col >= coerced_value - if op == "le": return col <= coerced_value + if op == "eq": + return col == coerced_value + if op == "ne": + return col != coerced_value + if op == "gt": + return col > coerced_value + if op == "lt": + return col < coerced_value + if op == "ge": + return col >= coerced_value + if op == "le": + return col <= coerced_value if op == "in": return col.in_(coerced_value if isinstance(coerced_value, (list, tuple)) else [coerced_value]) diff --git a/src/zcore/utils/helpers.py b/src/zcore/utils/helpers.py index eb8d36d..51ecb19 100644 --- a/src/zcore/utils/helpers.py +++ b/src/zcore/utils/helpers.py @@ -59,7 +59,7 @@ def default(self, obj: Any) -> Any: if isinstance(obj, (datetime, date, time)): return obj.isoformat() if isinstance(obj, Decimal): - return float(obj) + return str(obj) try: return super().default(obj) except TypeError: diff --git a/src/zcore/web/projection.py b/src/zcore/web/projection.py index 85dd3cc..d654255 100644 --- a/src/zcore/web/projection.py +++ b/src/zcore/web/projection.py @@ -14,31 +14,28 @@ class Zchema(BaseModel): """Unified, domain-aware security schema base class. - Subclasses specify their unique database domain mapping via the `__db_name__` + Subclasses specify their unique database domain mapping via the `__model__` class attribute. This enables contextual, recursive pruning across schema generation, input validation, and response serialization. """ - __db_name__: ClassVar[Optional[str]] = None + __model__: ClassVar[Optional[str]] = None @classmethod def _get_relative_restricted_paths(cls) -> Set[str]: """Extract and normalize restricted field paths mapped to this schema's domain.""" - db_name = getattr(cls, "__db_name__", None) - if not db_name: + model_name = getattr(cls, "__model__", None) + if not model_name: return set() - prefix = f"{db_name}." + prefix = f"{model_name}." restricted = get_restricted_fields() relative_paths = set() for path in restricted: - # Remove common resource prefix for robust backwards compatibility - normalized = path.replace("resource.", "") - if normalized.startswith(prefix): - relative_paths.add(normalized[len(prefix):]) - elif normalized == db_name: - # If the entire domain is restricted, restrict all attributes + if path.startswith(prefix): + relative_paths.add(path[len(prefix):]) + elif path == model_name: relative_paths.add("*") return relative_paths @@ -80,7 +77,7 @@ def _prune_data(cls, data: Any, relative_paths: Set[str]) -> Any: def __get_pydantic_json_schema__( cls, core_schema: Any, handler: Any ) -> dict[str, Any]: - """Level 1: Customize dynamic JSON schema generation under context boundaries.""" + """Customize dynamic JSON schema generation under context boundaries.""" json_schema = handler(core_schema) relative_paths = cls._get_relative_restricted_paths() if not relative_paths: @@ -121,7 +118,7 @@ def prune_schema(schema: dict[str, Any], paths: Set[str]) -> None: @model_validator(mode="before") @classmethod def filter_restricted_inputs(cls, data: Any) -> Any: - """Level 2: Silently strip restricted fields from input payloads to prevent Mass Assignment.""" + """Silently strip restricted fields from input payloads to prevent Mass Assignment.""" relative_paths = cls._get_relative_restricted_paths() if not relative_paths or data is None: return data @@ -133,7 +130,7 @@ def filter_restricted_inputs(cls, data: Any) -> Any: @model_serializer(mode="wrap") def secure_serializer(self, handler: Any) -> Any: - """Level 3: Securely intercept serialization to prune restricted attributes from response.""" + """Securely intercept serialization to prune restricted attributes from response.""" serialized = handler(self) relative_paths = self._get_relative_restricted_paths() if not relative_paths or serialized is None: diff --git a/tests/test_utils.py b/tests/test_utils.py index 2a46483..5b04314 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import datetime from decimal import Decimal import uuid -from typing import Any, Type +from typing import Any import pytest from zcore.exceptions.base import ValidationError @@ -29,11 +29,11 @@ ), ( Decimal("123.45"), - "123.45" + '"123.45"' ), ( {"user_id": uuid.UUID("12345678-1234-5678-1234-567812345678"), "balance": Decimal("9.99")}, - '{"user_id": "12345678-1234-5678-1234-567812345678", "balance": 9.99}' + '{"user_id": "12345678-1234-5678-1234-567812345678", "balance": "9.99"}' ) ] ) diff --git a/tests/test_web.py b/tests/test_web.py index 468f87f..db9510b 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -31,7 +31,7 @@ class DummyUpdate(BaseModel): name: str class DummyOut(Zchema): - __db_name__ = "dummy" + __model__ = "dummy" id: str name: str password: str = ""