Fixing string padding in import

This commit is contained in:
Ian Roddis committed 2026-07-17 15:27:15 -03:00
1 parent a0bd7f2c27
commit aca15db9bf
3 files changed
+21 -43

No files matched your search

+5
View File
@@ -22,6 +22,11 @@ migrate:
fmt: fmt:
uv run black . uv run black .
clean:
rm app.db
uv run alembic upgrade head
uv run python -m scripts.seed
docker: docker:
mkdir -p artifacts mkdir -p artifacts
docker build -t $(IMAGE_NAME):$(VERSION) . docker build -t $(IMAGE_NAME):$(VERSION) .
+6 -17
View File
@@ -21,14 +21,9 @@ async def get(db: AsyncSession, inventory_id: str) -> ImportedPart | None:
return await db.get(ImportedPart, inventory_id) return await db.get(ImportedPart, inventory_id)
async def list_all( async def list_all(db: AsyncSession, skip: int = 0, limit: int = 500) -> list[ImportedPart]:
db: AsyncSession, skip: int = 0, limit: int = 500
) -> list[ImportedPart]:
result = await db.execute( result = await db.execute(
select(ImportedPart) select(ImportedPart).order_by(ImportedPart.InventoryID).offset(skip).limit(limit)
.order_by(ImportedPart.InventoryID)
.offset(skip)
.limit(limit)
) )
return list(result.scalars().all()) return list(result.scalars().all())
@@ -42,9 +37,7 @@ async def list_filtered(
q = select(ImportedPart) q = select(ImportedPart)
if description.strip(): if description.strip():
q = q.where(ImportedPart.Description.ilike(f"%{description.strip()}%")) q = q.where(ImportedPart.Description.ilike(f"%{description.strip()}%"))
result = await db.execute( result = await db.execute(q.order_by(ImportedPart.InventoryID).offset(skip).limit(limit))
q.order_by(ImportedPart.InventoryID).offset(skip).limit(limit)
)
return list(result.scalars().all()) return list(result.scalars().all())
@@ -85,7 +78,7 @@ async def upsert_bulk(db: AsyncSession, records: list[dict]) -> tuple[int, int]:
return 0, 0 return 0, 0
clean = [ clean = [
{k: str(v) if v is not None else None for k, v in r.items() if k in _FIELDS} {k: str(v).strip() if v is not None else None for k, v in r.items() if k in _FIELDS}
for r in records for r in records
if r.get("InventoryID") if r.get("InventoryID")
] ]
@@ -95,9 +88,7 @@ async def upsert_bulk(db: AsyncSession, records: list[dict]) -> tuple[int, int]:
incoming_ids = {r["InventoryID"] for r in clean} incoming_ids = {r["InventoryID"] for r in clean}
result = await db.execute( result = await db.execute(
select(ImportedPart.InventoryID).where( select(ImportedPart.InventoryID).where(ImportedPart.InventoryID.in_(incoming_ids))
ImportedPart.InventoryID.in_(incoming_ids)
)
) )
existing_ids = {row[0] for row in result.all()} existing_ids = {row[0] for row in result.all()}
@@ -112,9 +103,7 @@ async def upsert_bulk(db: AsyncSession, records: list[dict]) -> tuple[int, int]:
vals = {k: v for k, v in record.items() if k != "InventoryID"} vals = {k: v for k, v in record.items() if k != "InventoryID"}
if vals: if vals:
await db.execute( await db.execute(
update(ImportedPart) update(ImportedPart).where(ImportedPart.InventoryID == inv_id).values(vals)
.where(ImportedPart.InventoryID == inv_id)
.values(vals)
) )
await db.commit() await db.commit()
+10 -26
View File
@@ -109,9 +109,7 @@ async def _get_or_create_attachment_point_name(db: AsyncSession, name: str):
return apn return apn
async def _add_attachment_point( async def _add_attachment_point(db: AsyncSession, ap: AugmentedPart, ip: ImportedPart) -> None:
db: AsyncSession, ap: AugmentedPart, ip: ImportedPart
) -> None:
"""Infer and create an attachment point for a newly-synced AugmentedPart. """Infer and create an attachment point for a newly-synced AugmentedPart.
Preference order: TBXThreadAttribute (ISO/UNC thread) then ProductODmm/in Preference order: TBXThreadAttribute (ISO/UNC thread) then ProductODmm/in
@@ -127,9 +125,7 @@ async def _add_attachment_point(
if kind == "iso": if kind == "iso":
depth_mm = _parse_float(ip.ProductMaxThreadDepthmm) depth_mm = _parse_float(ip.ProductMaxThreadDepthmm)
thread = await iso_thread_crud.get_by_values( thread = await iso_thread_crud.get_by_values(db, depth_mm=depth_mm, **fields)
db, depth_mm=depth_mm, **fields
)
if thread is None: if thread is None:
thread = await iso_thread_crud.create(db, depth_mm=depth_mm, **fields) thread = await iso_thread_crud.create(db, depth_mm=depth_mm, **fields)
await attachment_point_crud.create( await attachment_point_crud.create(
@@ -140,9 +136,7 @@ async def _add_attachment_point(
) )
elif kind == "unc": elif kind == "unc":
depth_in = _parse_float(ip.ProductMaxThreadDepthin) depth_in = _parse_float(ip.ProductMaxThreadDepthin)
thread = await unc_thread_crud.get_by_values( thread = await unc_thread_crud.get_by_values(db, depth_in=depth_in, **fields)
db, depth_in=depth_in, **fields
)
if thread is None: if thread is None:
thread = await unc_thread_crud.create(db, depth_in=depth_in, **fields) thread = await unc_thread_crud.create(db, depth_in=depth_in, **fields)
await attachment_point_crud.create( await attachment_point_crud.create(
@@ -201,7 +195,7 @@ async def sync_one(
manufacturer = await manufacturer_crud.create(db, name="Armament", url="") manufacturer = await manufacturer_crud.create(db, name="Armament", url="")
ap = AugmentedPart( ap = AugmentedPart(
sku=None, sku=ip.InventoryID,
description=description, description=description,
colour_id=colour.id, colour_id=colour.id,
manufacturer_id=manufacturer.id, manufacturer_id=manufacturer.id,
@@ -240,9 +234,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
# ------------------------------------------------------------------ phase 1 # ------------------------------------------------------------------ phase 1
# IDs referenced in existing associations # IDs referenced in existing associations
result = await db.execute( result = await db.execute(select(augmented_part_imported_parts.c.imported_part_id).distinct())
select(augmented_part_imported_parts.c.imported_part_id).distinct()
)
all_linked_ids: set[str] = {row[0] for row in result.all()} all_linked_ids: set[str] = {row[0] for row in result.all()}
# IDs that still exist in imported_parts # IDs that still exist in imported_parts
@@ -256,9 +248,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
# Which AugmentedParts are affected? # Which AugmentedParts are affected?
result = await db.execute( result = await db.execute(
select(augmented_part_imported_parts.c.augmented_part_id).where( select(augmented_part_imported_parts.c.augmented_part_id).where(
augmented_part_imported_parts.c.imported_part_id.in_( augmented_part_imported_parts.c.imported_part_id.in_(orphaned_import_ids)
orphaned_import_ids
)
) )
) )
orphaned_ap_ids: set[int] = {row[0] for row in result.all()} orphaned_ap_ids: set[int] = {row[0] for row in result.all()}
@@ -266,9 +256,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
# Remove the stale association rows # Remove the stale association rows
await db.execute( await db.execute(
augmented_part_imported_parts.delete().where( augmented_part_imported_parts.delete().where(
augmented_part_imported_parts.c.imported_part_id.in_( augmented_part_imported_parts.c.imported_part_id.in_(orphaned_import_ids)
orphaned_import_ids
)
) )
) )
await db.flush() await db.flush()
@@ -291,9 +279,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
# ------------------------------------------------------------------ phase 2 # ------------------------------------------------------------------ phase 2
# Re-query linked IDs after phase 1 cleanup # Re-query linked IDs after phase 1 cleanup
result = await db.execute( result = await db.execute(select(augmented_part_imported_parts.c.imported_part_id).distinct())
select(augmented_part_imported_parts.c.imported_part_id).distinct()
)
now_linked_ids: set[str] = {row[0] for row in result.all()} now_linked_ids: set[str] = {row[0] for row in result.all()}
unlinked_ids = current_ids - now_linked_ids unlinked_ids = current_ids - now_linked_ids
@@ -301,9 +287,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
skipped = 0 skipped = 0
if not unlinked_ids: if not unlinked_ids:
return AugmentedPartSyncResult( return AugmentedPartSyncResult(created=created, deleted=deleted, skipped=skipped)
created=created, deleted=deleted, skipped=skipped
)
result = await db.execute( result = await db.execute(
select(ImportedPart).where(ImportedPart.InventoryID.in_(unlinked_ids)) select(ImportedPart).where(ImportedPart.InventoryID.in_(unlinked_ids))
@@ -331,7 +315,7 @@ async def sync(db: AsyncSession) -> AugmentedPartSyncResult:
colour = await part_colour_crud.create(db, name=colour_name) colour = await part_colour_crud.create(db, name=colour_name)
ap = AugmentedPart( ap = AugmentedPart(
sku=None, sku=ip.InventoryID,
description=description, description=description,
colour_id=colour.id, colour_id=colour.id,
manufacturer_id=manufacturer.id, manufacturer_id=manufacturer.id,