From 291416544e5d6e9cdbbebabb1aab041c902f9ca9 Mon Sep 17 00:00:00 2001 From: Ian Roddis <31021769+iroddis@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:52:21 -0300 Subject: [PATCH] Adding OData Import --- .claude/settings.local.json | 3 +- .../7be5ef599d3f_add_imported_parts_tables.py | 29 +++++ app/api/imported_parts.py | 120 ++++++++++++++++++ app/api/router.py | 3 +- app/crud/imported_part.py | 74 +++++++++++ app/crud/imported_part_config.py | 34 +++++ app/models/__init__.py | 4 + app/models/imported_part.py | 17 +++ app/models/imported_part_config.py | 15 +++ app/schemas/imported_part.py | 34 +++++ app/services/odata.py | 24 ++++ app/templates/_nav.html | 3 + .../admin/_imported_parts_table.html | 44 +++++++ app/templates/admin/imported_parts.html | 41 ++++++ app/web/admin.py | 97 ++++++++++++++ pyproject.toml | 1 + scripts/seed.py | 5 +- uv.lock | 2 + 18 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/7be5ef599d3f_add_imported_parts_tables.py create mode 100644 app/api/imported_parts.py create mode 100644 app/crud/imported_part.py create mode 100644 app/crud/imported_part_config.py create mode 100644 app/models/imported_part.py create mode 100644 app/models/imported_part_config.py create mode 100644 app/schemas/imported_part.py create mode 100644 app/services/odata.py create mode 100644 app/templates/admin/_imported_parts_table.html create mode 100644 app/templates/admin/imported_parts.html diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 6d6b5ee..6b5d8cb 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -1,7 +1,8 @@ { "permissions": { "allow": [ - "Bash(uv run *)" + "Bash(uv run *)", + "Bash(uv sync *)" ] } } diff --git a/alembic/versions/7be5ef599d3f_add_imported_parts_tables.py b/alembic/versions/7be5ef599d3f_add_imported_parts_tables.py new file mode 100644 index 0000000..402fe83 --- /dev/null +++ b/alembic/versions/7be5ef599d3f_add_imported_parts_tables.py @@ -0,0 +1,29 @@ +"""add_imported_parts_tables + +Revision ID: 7be5ef599d3f +Revises: b60d0aa3ceda +Create Date: 2026-06-25 15:29:20.685050 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '7be5ef599d3f' +down_revision: Union[str, None] = 'b60d0aa3ceda' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/app/api/imported_parts.py b/app/api/imported_parts.py new file mode 100644 index 0000000..a02fb88 --- /dev/null +++ b/app/api/imported_parts.py @@ -0,0 +1,120 @@ +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.database import get_db +from app.crud import audit as audit_crud +from app.crud import imported_part as imported_part_crud +from app.crud import imported_part_config as config_crud +from app.deps import require_permission +from app.models.user import User +from app.schemas.imported_part import ( + ImportedPartConfigRead, + ImportedPartConfigUpdate, + ImportedPartRead, + SyncResult, +) +from app.services import odata as odata_service + +router = APIRouter() + + +@router.get("", response_model=list[ImportedPartRead]) +async def list_imported_parts( + skip: int = 0, + limit: int = 500, + db: AsyncSession = Depends(get_db), + _: User = Depends(require_permission("imported_parts:read")), +): + return await imported_part_crud.list_all(db, skip=skip, limit=limit) + + +@router.get("/config", response_model=ImportedPartConfigRead) +async def get_config( + db: AsyncSession = Depends(get_db), + _: User = Depends(require_permission("imported_parts:write")), +): + config = await config_crud.get(db) + if config is None: + return ImportedPartConfigRead(odata_url=None, username=None, has_password=False) + return ImportedPartConfigRead( + odata_url=config.odata_url, + username=config.username, + has_password=bool(config.password), + ) + + +@router.patch("/config", response_model=ImportedPartConfigRead) +async def update_config( + payload: ImportedPartConfigUpdate, + db: AsyncSession = Depends(get_db), + actor: User = Depends(require_permission("imported_parts:write")), +): + current = await config_crud.get(db) + # Keep existing password if not provided in the update + new_password = payload.password if payload.password is not None else (current.password if current else None) + config = await config_crud.upsert( + db, + odata_url=payload.odata_url, + username=payload.username, + password=new_password, + ) + await audit_crud.log( + db, actor=actor, action="update", resource_type="ImportedPartConfig", + resource_id=1, + description="Updated ImportedPart OData configuration", + ) + return ImportedPartConfigRead( + odata_url=config.odata_url, + username=config.username, + has_password=bool(config.password), + ) + + +@router.post("/refresh", response_model=SyncResult) +async def refresh_imported_parts( + db: AsyncSession = Depends(get_db), + actor: User = Depends(require_permission("imported_parts:write")), +): + config = await config_crud.get(db) + if not config or not config.odata_url: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="OData URL is not configured.", + ) + records = await odata_service.fetch(config.odata_url, config.username, config.password) + created, updated = await imported_part_crud.upsert_bulk(db, records) + await audit_crud.log( + db, actor=actor, action="update", resource_type="ImportedPart", + resource_id=None, + description=f"OData sync: {created} created, {updated} updated ({len(records)} total records)", + ) + return SyncResult(created=created, updated=updated, total=len(records)) + + +@router.get("/{inventory_id}", response_model=ImportedPartRead) +async def get_imported_part( + inventory_id: str, + db: AsyncSession = Depends(get_db), + _: User = Depends(require_permission("imported_parts:read")), +): + part = await imported_part_crud.get(db, inventory_id) + if part is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Imported part not found") + return part + + +@router.delete("/{inventory_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_imported_part( + inventory_id: str, + db: AsyncSession = Depends(get_db), + actor: User = Depends(require_permission("imported_parts:delete")), +): + part = await imported_part_crud.get(db, inventory_id) + if part is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Imported part not found") + await imported_part_crud.delete(db, part) + await audit_crud.log( + db, actor=actor, action="delete", resource_type="ImportedPart", + resource_id=None, + description=audit_crud.describe_delete("ImportedPart", 0, inventory_id), + ) diff --git a/app/api/router.py b/app/api/router.py index f985e69..72b0f10 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,6 +1,6 @@ from fastapi import APIRouter -from app.api import audit_log, auth, iso_threads, manufacturers, part_types, roles, scope_models, sub_part_types, unc_threads, users +from app.api import audit_log, auth, imported_parts, iso_threads, manufacturers, part_types, roles, scope_models, sub_part_types, unc_threads, users api_router = APIRouter() api_router.include_router(auth.router, prefix="/auth", tags=["auth"]) @@ -12,4 +12,5 @@ api_router.include_router(manufacturers.router, prefix="/manufacturers", tags=[" api_router.include_router(scope_models.router, prefix="/scope-models", tags=["scope-models"]) api_router.include_router(part_types.router, prefix="/part-types", tags=["part-types"]) api_router.include_router(sub_part_types.router, prefix="/sub-part-types", tags=["sub-part-types"]) +api_router.include_router(imported_parts.router, prefix="/imported-parts", tags=["imported-parts"]) api_router.include_router(audit_log.router, prefix="/audit-log", tags=["audit-log"]) diff --git a/app/crud/imported_part.py b/app/crud/imported_part.py new file mode 100644 index 0000000..3538de3 --- /dev/null +++ b/app/crud/imported_part.py @@ -0,0 +1,74 @@ +from sqlalchemy import insert, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.imported_part import ImportedPart + +_FIELDS = frozenset({ + "InventoryID", "Description", "TBXColour", + "ProductMaxThreadDepthmm", "ProductMaxThreadDepthin", + "ProductODmm", "ProductODin", "TBXThreadAttribute", +}) + + +async def get(db: AsyncSession, inventory_id: str) -> ImportedPart | None: + return await db.get(ImportedPart, inventory_id) + + +async def list_all( + db: AsyncSession, skip: int = 0, limit: int = 500 +) -> list[ImportedPart]: + result = await db.execute( + select(ImportedPart).order_by(ImportedPart.InventoryID).offset(skip).limit(limit) + ) + return list(result.scalars().all()) + + +async def count(db: AsyncSession) -> int: + from sqlalchemy import func + result = await db.execute(select(func.count()).select_from(ImportedPart)) + return result.scalar_one() + + +async def delete(db: AsyncSession, part: ImportedPart) -> None: + await db.delete(part) + await db.commit() + + +async def upsert_bulk(db: AsyncSession, records: list[dict]) -> tuple[int, int]: + """Insert-or-update records keyed on InventoryID. Returns (created, updated).""" + if not records: + return 0, 0 + + clean = [ + {k: str(v) if v is not None else None for k, v in r.items() if k in _FIELDS} + for r in records + if r.get("InventoryID") + ] + if not clean: + return 0, 0 + + incoming_ids = {r["InventoryID"] for r in clean} + + result = await db.execute( + select(ImportedPart.InventoryID).where(ImportedPart.InventoryID.in_(incoming_ids)) + ) + existing_ids = {row[0] for row in result.all()} + + to_insert = [r for r in clean if r["InventoryID"] not in existing_ids] + to_update = [r for r in clean if r["InventoryID"] in existing_ids] + + if to_insert: + await db.execute(insert(ImportedPart), to_insert) + + for record in to_update: + inv_id = record["InventoryID"] + vals = {k: v for k, v in record.items() if k != "InventoryID"} + if vals: + await db.execute( + update(ImportedPart) + .where(ImportedPart.InventoryID == inv_id) + .values(vals) + ) + + await db.commit() + return len(to_insert), len(to_update) diff --git a/app/crud/imported_part_config.py b/app/crud/imported_part_config.py new file mode 100644 index 0000000..42faf03 --- /dev/null +++ b/app/crud/imported_part_config.py @@ -0,0 +1,34 @@ +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.imported_part_config import ImportedPartConfig + +_SINGLETON_ID = 1 + + +async def get(db: AsyncSession) -> ImportedPartConfig | None: + return await db.get(ImportedPartConfig, _SINGLETON_ID) + + +async def upsert( + db: AsyncSession, + *, + odata_url: str | None, + username: str | None, + password: str | None, +) -> ImportedPartConfig: + config = await db.get(ImportedPartConfig, _SINGLETON_ID) + if config is None: + config = ImportedPartConfig( + id=_SINGLETON_ID, + odata_url=odata_url, + username=username, + password=password, + ) + db.add(config) + else: + config.odata_url = odata_url + config.username = username + config.password = password + await db.commit() + await db.refresh(config) + return config diff --git a/app/models/__init__.py b/app/models/__init__.py index 75d937f..da7ef11 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -2,6 +2,8 @@ from app.models.associations import role_permissions, user_roles from app.models.audit_log import AuditLog +from app.models.imported_part import ImportedPart +from app.models.imported_part_config import ImportedPartConfig from app.models.iso_thread import IsoThread from app.models.manufacturer import Manufacturer from app.models.part_type import PartType @@ -15,6 +17,8 @@ from app.models.user import User __all__ = [ "AuditLog", + "ImportedPart", + "ImportedPartConfig", "IsoThread", "Manufacturer", "PartType", diff --git a/app/models/imported_part.py b/app/models/imported_part.py new file mode 100644 index 0000000..79c96eb --- /dev/null +++ b/app/models/imported_part.py @@ -0,0 +1,17 @@ +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.database import Base + + +class ImportedPart(Base): + __tablename__ = "imported_parts" + + InventoryID: Mapped[str] = mapped_column(String, primary_key=True) + Description: Mapped[str | None] = mapped_column(nullable=True, default=None) + TBXColour: Mapped[str | None] = mapped_column(nullable=True, default=None) + ProductMaxThreadDepthmm: Mapped[str | None] = mapped_column(nullable=True, default=None) + ProductMaxThreadDepthin: Mapped[str | None] = mapped_column(nullable=True, default=None) + ProductODmm: Mapped[str | None] = mapped_column(nullable=True, default=None) + ProductODin: Mapped[str | None] = mapped_column(nullable=True, default=None) + TBXThreadAttribute: Mapped[str | None] = mapped_column(nullable=True, default=None) diff --git a/app/models/imported_part_config.py b/app/models/imported_part_config.py new file mode 100644 index 0000000..68ec5a0 --- /dev/null +++ b/app/models/imported_part_config.py @@ -0,0 +1,15 @@ +from sqlalchemy import String +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.database import Base + + +class ImportedPartConfig(Base): + """Singleton (id=1) storing the OData source URL and credentials.""" + + __tablename__ = "imported_part_config" + + id: Mapped[int] = mapped_column(primary_key=True, default=1) + odata_url: Mapped[str | None] = mapped_column(String, nullable=True, default=None) + username: Mapped[str | None] = mapped_column(String, nullable=True, default=None) + password: Mapped[str | None] = mapped_column(String, nullable=True, default=None) diff --git a/app/schemas/imported_part.py b/app/schemas/imported_part.py new file mode 100644 index 0000000..47530d0 --- /dev/null +++ b/app/schemas/imported_part.py @@ -0,0 +1,34 @@ +from pydantic import BaseModel, ConfigDict + + +class ImportedPartRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + InventoryID: str + Description: str | None + TBXColour: str | None + ProductMaxThreadDepthmm: str | None + ProductMaxThreadDepthin: str | None + ProductODmm: str | None + ProductODin: str | None + TBXThreadAttribute: str | None + + +class ImportedPartConfigRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + odata_url: str | None + username: str | None + has_password: bool + + +class ImportedPartConfigUpdate(BaseModel): + odata_url: str | None = None + username: str | None = None + password: str | None = None + + +class SyncResult(BaseModel): + created: int + updated: int + total: int diff --git a/app/services/odata.py b/app/services/odata.py new file mode 100644 index 0000000..19e66ad --- /dev/null +++ b/app/services/odata.py @@ -0,0 +1,24 @@ +"""Fetch records from an OData v4 JSON endpoint, following pagination.""" + +import httpx + + +async def fetch( + url: str, + username: str | None = None, + password: str | None = None, +) -> list[dict]: + auth = (username, password) if username and password else None + records: list[dict] = [] + next_url: str | None = url + + headers= {"Accept": "application/json" } + async with httpx.AsyncClient(timeout=30.0) as client: + while next_url: + response = await client.get(next_url, auth=auth, headers=headers) + response.raise_for_status() + body = response.json() + records.extend(body.get("value", [])) + next_url = body.get("@odata.nextLink") + + return records diff --git a/app/templates/_nav.html b/app/templates/_nav.html index 6404c64..aac8a7b 100644 --- a/app/templates/_nav.html +++ b/app/templates/_nav.html @@ -26,6 +26,9 @@ {% if user.is_superuser or "sub_part_types:read" in user.permissions %} Sub Part Types {% endif %} + {% if user.is_superuser or "imported_parts:read" in user.permissions %} + Imported Parts + {% endif %} {% if user.is_superuser or "audit_log:read" in user.permissions %} Audit Log {% endif %} diff --git a/app/templates/admin/_imported_parts_table.html b/app/templates/admin/_imported_parts_table.html new file mode 100644 index 0000000..5007039 --- /dev/null +++ b/app/templates/admin/_imported_parts_table.html @@ -0,0 +1,44 @@ +
+ {% if sync_result %} +

{{ sync_result }}

+ {% endif %} + {% if sync_error %} +

{{ sync_error }}

+ {% endif %} + +
+

+ Showing {{ parts|length }} record{{ '' if parts|length == 1 else 's' }}{% if total and total > parts|length %} of {{ total }} total{% endif %}. +

+ + + + + + + + + + + + + + + {% for p in parts %} + + + + + + + + + + + {% else %} + + {% endfor %} + +
InventoryIDDescriptionTBX ColourMax Thread Depth (mm)Max Thread Depth (in)OD (mm)OD (in)Thread Attribute
{{ p.InventoryID }}{{ p.Description or '—' }}{{ p.TBXColour or '—' }}{{ p.ProductMaxThreadDepthmm or '—' }}{{ p.ProductMaxThreadDepthin or '—' }}{{ p.ProductODmm or '—' }}{{ p.ProductODin or '—' }}{{ p.TBXThreadAttribute or '—' }}
No imported parts. Use Refresh to sync from OData.
+
+
diff --git a/app/templates/admin/imported_parts.html b/app/templates/admin/imported_parts.html new file mode 100644 index 0000000..c966694 --- /dev/null +++ b/app/templates/admin/imported_parts.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} +{% block title %}Imported Parts · {{ app_name }}{% endblock %} +{% block body %} +{% include "_nav.html" %} +
+

Imported Parts

+ +
+

OData Configuration

+
+ + + + +
+
+
+ +
+ + Loading… +
+ + {% include "admin/_imported_parts_table.html" %} +
+{% endblock %} diff --git a/app/web/admin.py b/app/web/admin.py index 2d16fcd..b3dbf79 100644 --- a/app/web/admin.py +++ b/app/web/admin.py @@ -6,8 +6,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.database import get_db from app.crud import audit as audit_crud +from app.crud import imported_part as imported_part_crud +from app.crud import imported_part_config as imported_part_config_crud from app.crud import iso_thread as iso_thread_crud from app.crud import sub_part_type as sub_part_type_crud +from app.services import odata as odata_service from app.crud import manufacturer as manufacturer_crud from app.crud import part_type as part_type_crud from app.crud import role as role_crud @@ -1007,6 +1010,100 @@ async def delete_sub_part_type( return Response(status_code=status.HTTP_200_OK) +# --- Imported Parts ---------------------------------------------------------- + + +@router.get("/imported-parts", response_class=HTMLResponse) +async def imported_parts_page( + request: Request, + db: AsyncSession = Depends(get_db), + user: User = Depends(web_require("imported_parts:read")), +): + config = await imported_part_config_crud.get(db) + config_view = { + "odata_url": config.odata_url if config else None, + "username": config.username if config else None, + "has_password": bool(config and config.password), + } + parts = await imported_part_crud.list_all(db) + total = await imported_part_crud.count(db) + return templates.TemplateResponse( + request, + "admin/imported_parts.html", + {"user": user, "config": config_view, "parts": parts, "total": total}, + ) + + +@router.post("/imported-parts/config", response_class=HTMLResponse) +async def save_imported_part_config( + request: Request, + odata_url: str = Form(""), + username: str = Form(""), + password: str = Form(""), + db: AsyncSession = Depends(get_db), + actor: User = Depends(web_require("imported_parts:write")), +): + current = await imported_part_config_crud.get(db) + # Keep existing password when field submitted blank + kept_password = password.strip() or (current.password if current else None) + await imported_part_config_crud.upsert( + db, + odata_url=odata_url.strip() or None, + username=username.strip() or None, + password=kept_password, + ) + await audit_crud.log( + db, actor=actor, action="update", resource_type="ImportedPartConfig", + resource_id=1, + description="Updated ImportedPart OData configuration", + ) + return HTMLResponse("Configuration saved.", status_code=200) + + +@router.post("/imported-parts/refresh", response_class=HTMLResponse) +async def refresh_imported_parts( + request: Request, + db: AsyncSession = Depends(get_db), + actor: User = Depends(web_require("imported_parts:write")), +): + config = await imported_part_config_crud.get(db) + if not config or not config.odata_url: + parts = await imported_part_crud.list_all(db) + total = await imported_part_crud.count(db) + return templates.TemplateResponse( + request, + "admin/_imported_parts_table.html", + {"parts": parts, "total": total, "sync_error": "OData URL is not configured."}, + ) + try: + records = await odata_service.fetch(config.odata_url, config.username, config.password) + created, updated = await imported_part_crud.upsert_bulk(db, records) + await audit_crud.log( + db, actor=actor, action="update", resource_type="ImportedPart", + resource_id=None, + description=f"OData sync: {created} created, {updated} updated ({len(records)} total records)", + ) + parts = await imported_part_crud.list_all(db) + total = await imported_part_crud.count(db) + return templates.TemplateResponse( + request, + "admin/_imported_parts_table.html", + { + "parts": parts, + "total": total, + "sync_result": f"Sync complete: {created} created, {updated} updated.", + }, + ) + except Exception as exc: + parts = await imported_part_crud.list_all(db) + total = await imported_part_crud.count(db) + return templates.TemplateResponse( + request, + "admin/_imported_parts_table.html", + {"parts": parts, "total": total, "sync_error": f"Sync failed: {exc}"}, + ) + + # --- Audit Log --------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index c97676c..4a8ce55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "pydantic-settings>=2.3", "pyjwt>=2.8", "bcrypt>=4.1", + "httpx>=0.27", ] [project.optional-dependencies] diff --git a/scripts/seed.py b/scripts/seed.py index 50150b6..457ffc3 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -47,13 +47,16 @@ PERMISSIONS = [ ("sub_part_types:read", "View sub part types"), ("sub_part_types:write", "Create and update sub part types"), ("sub_part_types:delete", "Delete sub part types"), + ("imported_parts:read", "View imported parts"), + ("imported_parts:write", "Sync and configure imported parts"), + ("imported_parts:delete", "Delete imported parts"), ("audit_log:read", "View audit log"), ] # Default roles -> permission names. "admin" gets everything. ROLES = { "admin": [name for name, _ in PERMISSIONS], # includes audit_log:read - "user": ["users:read", "iso_threads:read", "unc_threads:read", "manufacturers:read", "scope_models:read", "part_types:read", "sub_part_types:read"], + "user": ["users:read", "iso_threads:read", "unc_threads:read", "manufacturers:read", "scope_models:read", "part_types:read", "sub_part_types:read", "imported_parts:read"], } diff --git a/uv.lock b/uv.lock index a9b1180..46f8bfe 100644 --- a/uv.lock +++ b/uv.lock @@ -289,6 +289,7 @@ dependencies = [ { name = "asyncpg" }, { name = "bcrypt" }, { name = "fastapi" }, + { name = "httpx" }, { name = "jinja2" }, { name = "pydantic", extra = ["email"] }, { name = "pydantic-settings" }, @@ -315,6 +316,7 @@ requires-dist = [ { name = "bcrypt", specifier = ">=4.1" }, { name = "black", marker = "extra == 'dev'", specifier = ">=24" }, { name = "fastapi", specifier = ">=0.111" }, + { name = "httpx", specifier = ">=0.27" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27" }, { name = "jinja2", specifier = ">=3.1" }, { name = "pydantic", extras = ["email"], specifier = ">=2.7" },