Adding sync with imported parts

alembic/versions/bfbce324cd5f_add_augmented_part_imported_parts.py

@@ -0,0 +1,29 @@
+"""add augmented_part_imported_parts
+
+Revision ID: bfbce324cd5f
+Revises: 8bc411b5814e
+Create Date: 2026-07-02 12:46:27.493942
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'bfbce324cd5f'
+down_revision: Union[str, None] = '8bc411b5814e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

app/api/augmented_parts.py

@@ -9,8 +9,10 @@ from app.models.user import User
 from app.schemas.augmented_part import (
     AugmentedPartCreate,
     AugmentedPartRead,
+    AugmentedPartSyncResult,
     AugmentedPartUpdate,
 )
+from app.services import augmented_part_sync
 
 router = APIRouter()
 
@@ -49,6 +51,23 @@ async def create_augmented_part(
     return ap
 
 
+@router.post("/sync", response_model=AugmentedPartSyncResult)
+async def sync_augmented_parts(
+    db: AsyncSession = Depends(get_db),
+    actor: User = Depends(require_permission("augmented_parts:write")),
+):
+    result = await augmented_part_sync.sync(db)
+    await audit_crud.log(
+        db, actor=actor, action="update", resource_type="AugmentedPart",
+        resource_id=None,
+        description=(
+            f"Sync: {result.created} created, {result.deleted} deleted, "
+            f"{result.skipped} skipped (no type match)"
+        ),
+    )
+    return result
+
+
 @router.get("/{augmented_part_id}", response_model=AugmentedPartRead)
 async def get_augmented_part(
     augmented_part_id: int,

app/crud/manufacturer.py

@@ -8,6 +8,13 @@ async def get(db: AsyncSession, manufacturer_id: int) -> Manufacturer | None:
     return await db.get(Manufacturer, manufacturer_id)
 
 
+async def get_by_name(db: AsyncSession, name: str) -> Manufacturer | None:
+    result = await db.execute(
+        select(Manufacturer).where(Manufacturer.name == name)
+    )
+    return result.scalar_one_or_none()
+
+
 async def list_all(
     db: AsyncSession, skip: int = 0, limit: int = 100
 ) -> list[Manufacturer]:

app/models/__init__.py

@@ -1,6 +1,6 @@
 """Import all models so they register with ``Base.metadata``."""
 
-from app.models.associations import role_permissions, user_roles
+from app.models.associations import augmented_part_imported_parts, role_permissions, user_roles
 from app.models.audit_log import AuditLog
 from app.models.augmented_part import AugmentedPart
 from app.models.imported_part import ImportedPart
@@ -35,6 +35,7 @@ __all__ = [
     "Role",
     "UncThread",
     "User",
+    "augmented_part_imported_parts",
     "role_permissions",
     "user_roles",
 ]

app/models/associations.py

@@ -1,6 +1,6 @@
-"""Many-to-many association tables for RBAC."""
+"""Many-to-many association tables."""
 
-from sqlalchemy import Column, ForeignKey, Table
+from sqlalchemy import Column, ForeignKey, String, Table
 
 from app.core.database import Base
 
@@ -21,3 +21,19 @@ role_permissions = Table(
         primary_key=True,
     ),
 )
+
+augmented_part_imported_parts = Table(
+    "augmented_part_imported_parts",
+    Base.metadata,
+    Column(
+        "augmented_part_id",
+        ForeignKey("augmented_parts.id", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+    Column(
+        "imported_part_id",
+        String,
+        ForeignKey("imported_parts.InventoryID", ondelete="CASCADE"),
+        primary_key=True,
+    ),
+)

app/models/augmented_part.py

@@ -2,6 +2,8 @@ from sqlalchemy import ForeignKey, String
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.core.database import Base
+from app.models.associations import augmented_part_imported_parts
+from app.models.imported_part import ImportedPart
 from app.models.manufacturer import Manufacturer
 from app.models.part_colour import PartColour
 from app.models.part_type import PartType
@@ -31,6 +33,11 @@ class AugmentedPart(Base):
         ForeignKey("sub_part_types.id", ondelete="SET NULL"), nullable=True, default=None
     )
 
+    # linked imported parts — not eagerly loaded; used by sync service only
+    imported_parts: Mapped[list[ImportedPart]] = relationship(
+        secondary=augmented_part_imported_parts, lazy="select"
+    )
+
     colour: Mapped[PartColour] = relationship(lazy="selectin")
     manufacturer: Mapped[Manufacturer] = relationship(lazy="selectin")
     scope_model: Mapped[ScopeModel | None] = relationship(lazy="selectin")

app/schemas/augmented_part.py

@@ -31,6 +31,12 @@ class AugmentedPartUpdate(BaseModel):
     sub_part_type_id: int | None = None
 
 
+class AugmentedPartSyncResult(BaseModel):
+    created: int
+    deleted: int
+    skipped: int
+
+
 class AugmentedPartRead(AugmentedPartBase):
     model_config = ConfigDict(from_attributes=True)
 

app/services/augmented_part_sync.py

@@ -0,0 +1,158 @@
+"""Synchronise AugmentedParts against the current ImportedParts table.
+
+Algorithm
+---------
+Phase 1 – prune stale entries
+  • Find association rows whose imported_part_id no longer exists.
+  • Delete those rows.
+  • Delete any AugmentedPart that ends up with zero remaining associations
+    (i.e. it was created purely from now-deleted ImportedParts).
+
+Phase 2 – create entries for new ImportedParts
+  • Any ImportedPart not yet linked to an AugmentedPart gets one created.
+  • Description   ← ImportedPart.Description (fallback: InventoryID)
+  • Colour        ← ImportedPart.TBXColour   (fallback: "Unknown"; created if needed)
+  • Manufacturer  ← "Armament"               (created if needed)
+  • Model         ← null
+  • Type          ← first PartType whose name is a substring of the description
+                    (longest name wins; skipped if no match)
+  • SubType       ← null
+"""
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.crud import manufacturer as manufacturer_crud
+from app.crud import part_colour as part_colour_crud
+from app.crud import part_type as part_type_crud
+from app.models.associations import augmented_part_imported_parts
+from app.models.augmented_part import AugmentedPart
+from app.models.imported_part import ImportedPart
+from app.models.part_type import PartType
+from app.schemas.augmented_part import AugmentedPartSyncResult
+
+
+def _infer_part_type(description: str, part_types: list[PartType]) -> PartType | None:
+    """Return the best-matching PartType or None.
+
+    Sorted by name length descending so more specific types (e.g. "Flip Cover")
+    are preferred over shorter ones (e.g. "Scope").
+    """
+    desc_lower = description.lower()
+    for pt in sorted(part_types, key=lambda p: len(p.name), reverse=True):
+        if pt.name.lower() in desc_lower:
+            return pt
+    return None
+
+
+async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
+    # ------------------------------------------------------------------ phase 1
+
+    # IDs referenced in existing associations
+    result = await db.execute(
+        select(augmented_part_imported_parts.c.imported_part_id).distinct()
+    )
+    all_linked_ids: set[str] = {row[0] for row in result.all()}
+
+    # IDs that still exist in imported_parts
+    result = await db.execute(select(ImportedPart.InventoryID))
+    current_ids: set[str] = {row[0] for row in result.all()}
+
+    orphaned_import_ids = all_linked_ids - current_ids
+    deleted = 0
+
+    if orphaned_import_ids:
+        # Which AugmentedParts are affected?
+        result = await db.execute(
+            select(augmented_part_imported_parts.c.augmented_part_id).where(
+                augmented_part_imported_parts.c.imported_part_id.in_(orphaned_import_ids)
+            )
+        )
+        orphaned_ap_ids: set[int] = {row[0] for row in result.all()}
+
+        # Remove the stale association rows
+        await db.execute(
+            augmented_part_imported_parts.delete().where(
+                augmented_part_imported_parts.c.imported_part_id.in_(orphaned_import_ids)
+            )
+        )
+        await db.flush()
+
+        # Delete AugmentedParts that now have no remaining associations
+        for ap_id in orphaned_ap_ids:
+            result = await db.execute(
+                select(func.count())
+                .select_from(augmented_part_imported_parts)
+                .where(augmented_part_imported_parts.c.augmented_part_id == ap_id)
+            )
+            if result.scalar() == 0:
+                ap = await db.get(AugmentedPart, ap_id)
+                if ap is not None:
+                    await db.delete(ap)
+                    deleted += 1
+
+        await db.commit()
+
+    # ------------------------------------------------------------------ phase 2
+
+    # Re-query linked IDs after phase 1 cleanup
+    result = await db.execute(
+        select(augmented_part_imported_parts.c.imported_part_id).distinct()
+    )
+    now_linked_ids: set[str] = {row[0] for row in result.all()}
+    unlinked_ids = current_ids - now_linked_ids
+
+    created = 0
+    skipped = 0
+
+    if not unlinked_ids:
+        return AugmentedPartSyncResult(created=created, deleted=deleted, skipped=skipped)
+
+    result = await db.execute(
+        select(ImportedPart).where(ImportedPart.InventoryID.in_(unlinked_ids))
+    )
+    new_parts = list(result.scalars().all())
+
+    # Ensure "Armament" manufacturer exists
+    manufacturer = await manufacturer_crud.get_by_name(db, "Armament")
+    if manufacturer is None:
+        manufacturer = await manufacturer_crud.create(db, name="Armament", url="")
+
+    part_types = await part_type_crud.list_all(db)
+
+    for ip in new_parts:
+        description = (ip.Description or "").strip() or ip.InventoryID
+
+        part_type = _infer_part_type(description, part_types)
+        if part_type is None:
+            skipped += 1
+            continue
+
+        colour_name = (ip.TBXColour or "").strip() or "Unknown"
+        colour = await part_colour_crud.get_by_name(db, colour_name)
+        if colour is None:
+            colour = await part_colour_crud.create(db, name=colour_name)
+
+        ap = AugmentedPart(
+            sku=None,
+            description=description,
+            colour_id=colour.id,
+            manufacturer_id=manufacturer.id,
+            scope_model_id=None,
+            part_type_id=part_type.id,
+            sub_part_type_id=None,
+        )
+        db.add(ap)
+        await db.flush()
+
+        await db.execute(
+            augmented_part_imported_parts.insert().values(
+                augmented_part_id=ap.id,
+                imported_part_id=ip.InventoryID,
+            )
+        )
+        created += 1
+
+    await db.commit()
+
+    return AugmentedPartSyncResult(created=created, deleted=deleted, skipped=skipped)

app/templates/admin/_augmented_parts_table.html

@@ -1,4 +1,11 @@
-<div id="augmented-part-list" class="card">
+<div id="augmented-part-list">
+{% if sync_result %}
+  <p class="alert alert-success">{{ sync_result }}</p>
+{% endif %}
+{% if sync_error %}
+  <p class="alert alert-error">{{ sync_error }}</p>
+{% endif %}
+<div class="card">
   <table class="table">
     <thead>
       <tr>
@@ -22,3 +29,4 @@
     </tbody>
   </table>
 </div>
+</div>

app/templates/admin/augmented_parts.html

@@ -54,6 +54,14 @@
     </form>
   </div>
 
+  <div class="card">
+    <h2>Sync with Imported Parts</h2>
+    <p class="muted">Creates Augmented Part entries for new Imported Parts, and removes entries for deleted Imported Parts.</p>
+    <form hx-post="/admin/augmented-parts/sync" hx-target="#augmented-part-list" hx-swap="outerHTML">
+      <button type="submit" class="btn btn-primary">Sync now</button>
+    </form>
+  </div>
+
   {% include "admin/_augmented_parts_table.html" %}
 </main>
 {% endblock %}

app/web/admin.py

@@ -22,6 +22,7 @@ from app.crud import user as user_crud
 from app.deps import get_current_web_user
 from app.models.unc_thread import UncSeries
 from app.models.user import User
+from app.services import augmented_part_sync as augmented_part_sync_service
 from app.services import odata as odata_service
 from app.web.templates import templates
 
@@ -1182,6 +1183,42 @@ async def create_augmented_part(
     )
 
 
+@router.post("/augmented-parts/sync", response_class=HTMLResponse)
+async def sync_augmented_parts(
+    request: Request,
+    db: AsyncSession = Depends(get_db),
+    actor: User = Depends(web_require("augmented_parts:write")),
+):
+    try:
+        result = await augmented_part_sync_service.sync(db)
+        await audit_crud.log(
+            db, actor=actor, action="update", resource_type="AugmentedPart",
+            resource_id=None,
+            description=(
+                f"Sync: {result.created} created, {result.deleted} deleted, "
+                f"{result.skipped} skipped (no type match)"
+            ),
+        )
+        sync_result = (
+            f"Sync complete: {result.created} created, {result.deleted} deleted"
+            + (f", {result.skipped} skipped (type not recognised)" if result.skipped else "")
+            + "."
+        )
+        sync_error = None
+    except Exception as exc:
+        sync_result = None
+        sync_error = f"Sync failed: {exc}"
+    return templates.TemplateResponse(
+        request,
+        "admin/_augmented_parts_table.html",
+        {
+            "augmented_parts": await augmented_part_crud.list_all(db),
+            "sync_result": sync_result,
+            "sync_error": sync_error,
+        },
+    )
+
+
 @router.get("/augmented-parts/{augmented_part_id}/edit", response_class=HTMLResponse)
 async def edit_augmented_part_row(
     request: Request,