v2: Program research, Groq extraction, and workout tracking (REVIEW — not deployed)
Backend: - New models: ProgramCatalog, CatalogWorkout, CrawlQueue, WorkoutLog - Program.py: added catalog_id, current_week, current_day columns - program_research.py: ttp-crawler integration, Groq Llama 3.3 70B extraction, known sources map, validation - crawl_scheduler.py: priority queue with off-peak/weekend scheduling - catalog.py router: research, catalog browse, adopt, schedule, complete, progress endpoints - Points: 75pts/workout, 50pts weekly bonus, 200pts completion bonus - config.py: added groq_api_key setting - main.py: catalog router, background scheduler, v2 migration Frontend: - ProgramSearch.jsx: search, research submission, catalog browsing, adopt programs - ProgramDetail.jsx: schedule view, today's workout, completion modal, progress tracking - App.jsx: Programs nav tab, new routes - api.js: catalog and tracking API functions Config: - docker-compose.yml: GROQ_API_KEY env var NOT DEPLOYED — requires Groq API key and Scot review
This commit is contained in:
claude
parent
a81ca9c275
commit
93f24fff83
13 changed files with 1738 additions and 9 deletions
|
|
@ -21,6 +21,9 @@ class Settings(BaseSettings):
|
|||
polar_client_id: str = ""
|
||||
polar_client_secret: str = ""
|
||||
|
||||
# Groq (program extraction)
|
||||
groq_api_key: str = ""
|
||||
|
||||
# App
|
||||
base_url: str = "https://fitness.littlefake.com"
|
||||
timezone: str = "Asia/Bangkok"
|
||||
|
|
@ -31,3 +34,7 @@ class Settings(BaseSettings):
|
|||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
|
||||
|
||||
# Module-level shortcut used by services
|
||||
settings = get_settings()
|
||||
|
|
|
|||
|
|
@ -41,12 +41,29 @@ async def lifespan(app: FastAPI):
|
|||
except Exception as e:
|
||||
logger.warning(f"Resource seeding failed (non-fatal): {e}")
|
||||
|
||||
# Start background crawl queue scheduler
|
||||
crawl_task = None
|
||||
try:
|
||||
from app.services.crawl_scheduler import crawl_queue_loop
|
||||
crawl_task = asyncio.create_task(crawl_queue_loop())
|
||||
logger.info("Crawl queue scheduler started")
|
||||
except Exception as e:
|
||||
logger.warning(f"Crawl scheduler failed to start (non-fatal): {e}")
|
||||
|
||||
logger.info("Fitness Rewards backend started")
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
if crawl_task:
|
||||
crawl_task.cancel()
|
||||
try:
|
||||
await crawl_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
logger.info("Fitness Rewards backend shutting down")
|
||||
|
||||
|
||||
app = FastAPI(title="Fitness Rewards", version="0.1.0", lifespan=lifespan)
|
||||
app = FastAPI(title="Fitness Rewards", version="0.2.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
|
|
@ -64,6 +81,7 @@ from app.routers.food import router as food_router
|
|||
from app.routers.points import router as points_router, rewards_router
|
||||
from app.routers.integrations import router as integrations_router
|
||||
from app.routers.programs import router as programs_router
|
||||
from app.routers.catalog import router as catalog_router
|
||||
|
||||
app.include_router(auth_router)
|
||||
app.include_router(onboarding_router)
|
||||
|
|
@ -73,11 +91,12 @@ app.include_router(points_router)
|
|||
app.include_router(rewards_router)
|
||||
app.include_router(integrations_router)
|
||||
app.include_router(programs_router)
|
||||
app.include_router(catalog_router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok", "version": "0.1.0"}
|
||||
return {"status": "ok", "version": "0.2.0"}
|
||||
|
||||
|
||||
|
||||
|
|
@ -87,7 +106,7 @@ async def run_migrations():
|
|||
from sqlalchemy import text
|
||||
|
||||
async with engine.begin() as conn:
|
||||
# Add equipment_list JSONB column if missing
|
||||
# v1: Add equipment_list JSONB column if missing
|
||||
await conn.execute(text("""
|
||||
DO $$ BEGIN
|
||||
ALTER TABLE user_profiles ADD COLUMN IF NOT EXISTS equipment_list JSONB DEFAULT '[]';
|
||||
|
|
@ -95,6 +114,16 @@ async def run_migrations():
|
|||
END $$;
|
||||
"""))
|
||||
|
||||
# v2: Add catalog columns to programs table
|
||||
await conn.execute(text("""
|
||||
DO $$ BEGIN
|
||||
ALTER TABLE programs ADD COLUMN IF NOT EXISTS catalog_id UUID REFERENCES program_catalog(id);
|
||||
ALTER TABLE programs ADD COLUMN IF NOT EXISTS current_week INTEGER DEFAULT 1;
|
||||
ALTER TABLE programs ADD COLUMN IF NOT EXISTS current_day INTEGER DEFAULT 1;
|
||||
EXCEPTION WHEN undefined_table THEN NULL;
|
||||
END $$;
|
||||
"""))
|
||||
|
||||
|
||||
async def seed_resources():
|
||||
"""Seed the resource library with curated fitness programs."""
|
||||
|
|
@ -232,4 +261,4 @@ async def seed_resources():
|
|||
|
||||
for r in resources:
|
||||
db.add(r)
|
||||
await db.commit()
|
||||
await db.commit()
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ from app.models.points import PointRule, DailyTarget
|
|||
from app.models.reward import Reward, RewardRedemption
|
||||
from app.models.oauth import OAuthToken
|
||||
from app.models.resource import Resource
|
||||
from app.models.catalog import ProgramCatalog, CatalogWorkout, CrawlQueue, WorkoutLog
|
||||
|
||||
__all__ = [
|
||||
"User", "UserProfile", "Program", "ActivityLog", "FoodLog",
|
||||
"PointRule", "DailyTarget", "Reward", "RewardRedemption",
|
||||
"OAuthToken", "Resource",
|
||||
"ProgramCatalog", "CatalogWorkout", "CrawlQueue", "WorkoutLog",
|
||||
]
|
||||
|
|
|
|||
102
backend/app/models/catalog.py
Normal file
102
backend/app/models/catalog.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from sqlalchemy import String, Integer, Text, Boolean, DateTime, ForeignKey, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class ProgramCatalog(Base):
|
||||
"""Shared program definitions — one per program, reusable by all users."""
|
||||
__tablename__ = "program_catalog"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
name: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
slug: Mapped[str] = mapped_column(String(200), unique=True, nullable=False)
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
source_url: Mapped[str | None] = mapped_column(Text)
|
||||
duration_weeks: Mapped[int | None] = mapped_column(Integer)
|
||||
frequency_per_week: Mapped[int | None] = mapped_column(Integer)
|
||||
equipment: Mapped[dict] = mapped_column(JSONB, default=list)
|
||||
difficulty: Mapped[str | None] = mapped_column(String(20))
|
||||
category: Mapped[str | None] = mapped_column(String(50))
|
||||
progression_rules: Mapped[str | None] = mapped_column(Text)
|
||||
structured_data: Mapped[dict | None] = mapped_column(JSONB)
|
||||
crawl_status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
crawl_error: Mapped[str | None] = mapped_column(Text)
|
||||
crawled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
workouts: Mapped[list["CatalogWorkout"]] = relationship(
|
||||
back_populates="catalog", cascade="all, delete-orphan",
|
||||
order_by="CatalogWorkout.week_number, CatalogWorkout.day_number"
|
||||
)
|
||||
|
||||
|
||||
class CatalogWorkout(Base):
|
||||
"""Individual workout within a catalog program."""
|
||||
__tablename__ = "catalog_workouts"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
catalog_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("program_catalog.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
week_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
day_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
workout_name: Mapped[str | None] = mapped_column(String(200))
|
||||
exercises: Mapped[dict] = mapped_column(JSONB, nullable=False)
|
||||
notes: Mapped[str | None] = mapped_column(Text)
|
||||
rest_day: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
catalog: Mapped["ProgramCatalog"] = relationship(back_populates="workouts")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("catalog_id", "week_number", "day_number", name="uq_catalog_week_day"),
|
||||
)
|
||||
|
||||
|
||||
class CrawlQueue(Base):
|
||||
"""Job queue for program research crawls."""
|
||||
__tablename__ = "crawl_queue"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
catalog_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("program_catalog.id"), nullable=True
|
||||
)
|
||||
search_query: Mapped[str] = mapped_column(String(500), nullable=False)
|
||||
priority: Mapped[str] = mapped_column(String(10), default="low")
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
urls_to_crawl: Mapped[dict] = mapped_column(JSONB, default=list)
|
||||
crawled_content: Mapped[str | None] = mapped_column(Text)
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class WorkoutLog(Base):
|
||||
"""Per-user workout completion tracking against catalog workouts."""
|
||||
__tablename__ = "workout_logs"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("users.id"), nullable=False
|
||||
)
|
||||
program_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("programs.id"), nullable=False
|
||||
)
|
||||
catalog_workout_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("catalog_workouts.id"), nullable=False
|
||||
)
|
||||
completed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
exercises_completed: Mapped[dict | None] = mapped_column(JSONB)
|
||||
points_earned: Mapped[int] = mapped_column(Integer, default=0)
|
||||
notes: Mapped[str | None] = mapped_column(Text)
|
||||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import uuid
|
||||
from datetime import datetime, date, timezone
|
||||
from sqlalchemy import String, Text, Date, DateTime, ForeignKey
|
||||
from sqlalchemy import String, Integer, Text, Date, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from app.database import Base
|
||||
|
|
@ -22,4 +22,11 @@ class Program(Base):
|
|||
notes: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||
|
||||
# v2: Link to catalog program
|
||||
catalog_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("program_catalog.id"), nullable=True
|
||||
)
|
||||
current_week: Mapped[int] = mapped_column(Integer, default=1)
|
||||
current_day: Mapped[int] = mapped_column(Integer, default=1)
|
||||
|
||||
user: Mapped["User"] = relationship(back_populates="programs")
|
||||
|
|
|
|||
611
backend/app/routers/catalog.py
Normal file
611
backend/app/routers/catalog.py
Normal file
|
|
@ -0,0 +1,611 @@
|
|||
"""Catalog router — program research, browsing, adoption, and workout tracking."""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid as uuid_mod
|
||||
from datetime import date, timedelta, datetime, timezone
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.user import User
|
||||
from app.models.program import Program
|
||||
from app.models.catalog import ProgramCatalog, CatalogWorkout, CrawlQueue, WorkoutLog
|
||||
from app.services.program_research import slugify, find_urls_for_program
|
||||
from app.services.points_engine import log_activity_with_points
|
||||
from app.utils.auth import get_current_user
|
||||
|
||||
router = APIRouter(prefix="/api/programs", tags=["programs-v2"])
|
||||
|
||||
|
||||
# ── Schemas ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class ResearchRequest(BaseModel):
|
||||
name: str
|
||||
|
||||
class AdoptRequest(BaseModel):
|
||||
start_date: date
|
||||
|
||||
class CompleteWorkoutRequest(BaseModel):
|
||||
exercises_completed: list[dict] | None = None
|
||||
notes: str | None = None
|
||||
|
||||
|
||||
# ── Catalog Endpoints ───────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/catalog")
|
||||
async def list_catalog(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
q: str | None = None,
|
||||
):
|
||||
"""List all programs in catalog, optionally filtered by search query."""
|
||||
query = select(ProgramCatalog).order_by(ProgramCatalog.created_at.desc())
|
||||
if q:
|
||||
query = query.where(ProgramCatalog.name.ilike(f"%{q}%"))
|
||||
|
||||
result = await db.execute(query)
|
||||
programs = result.scalars().all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": str(p.id),
|
||||
"name": p.name,
|
||||
"slug": p.slug,
|
||||
"description": p.description,
|
||||
"duration_weeks": p.duration_weeks,
|
||||
"frequency_per_week": p.frequency_per_week,
|
||||
"equipment": p.equipment or [],
|
||||
"difficulty": p.difficulty,
|
||||
"category": p.category,
|
||||
"crawl_status": p.crawl_status,
|
||||
"source_url": p.source_url,
|
||||
}
|
||||
for p in programs
|
||||
]
|
||||
|
||||
|
||||
@router.get("/catalog/{catalog_id}")
|
||||
async def get_catalog_program(
|
||||
catalog_id: str,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Get full catalog program detail including workouts."""
|
||||
result = await db.execute(
|
||||
select(ProgramCatalog)
|
||||
.options(selectinload(ProgramCatalog.workouts))
|
||||
.where(ProgramCatalog.id == uuid_mod.UUID(catalog_id))
|
||||
)
|
||||
p = result.scalar_one_or_none()
|
||||
if not p:
|
||||
raise HTTPException(status_code=404, detail="Program not found in catalog")
|
||||
|
||||
workouts = []
|
||||
for w in p.workouts:
|
||||
workouts.append({
|
||||
"id": str(w.id),
|
||||
"week_number": w.week_number,
|
||||
"day_number": w.day_number,
|
||||
"workout_name": w.workout_name,
|
||||
"exercises": w.exercises,
|
||||
"notes": w.notes,
|
||||
"rest_day": w.rest_day,
|
||||
})
|
||||
|
||||
return {
|
||||
"id": str(p.id),
|
||||
"name": p.name,
|
||||
"description": p.description,
|
||||
"duration_weeks": p.duration_weeks,
|
||||
"frequency_per_week": p.frequency_per_week,
|
||||
"equipment": p.equipment or [],
|
||||
"difficulty": p.difficulty,
|
||||
"category": p.category,
|
||||
"progression_rules": p.progression_rules,
|
||||
"source_url": p.source_url,
|
||||
"crawl_status": p.crawl_status,
|
||||
"crawl_error": p.crawl_error,
|
||||
"workouts": workouts,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/research")
|
||||
async def research_program(
|
||||
req: ResearchRequest,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Submit a program name for research. Creates catalog entry + crawl job."""
|
||||
slug = slugify(req.name)
|
||||
|
||||
# Check if already in catalog
|
||||
result = await db.execute(
|
||||
select(ProgramCatalog).where(ProgramCatalog.slug == slug)
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing:
|
||||
return {
|
||||
"id": str(existing.id),
|
||||
"name": existing.name,
|
||||
"crawl_status": existing.crawl_status,
|
||||
"already_exists": True,
|
||||
}
|
||||
|
||||
# Find URLs
|
||||
urls = find_urls_for_program(req.name)
|
||||
|
||||
# Create catalog entry
|
||||
catalog = ProgramCatalog(
|
||||
name=req.name.strip(),
|
||||
slug=slug,
|
||||
crawl_status="pending",
|
||||
source_url=urls[0] if urls else None,
|
||||
)
|
||||
db.add(catalog)
|
||||
await db.flush()
|
||||
|
||||
# Create crawl job
|
||||
job = CrawlQueue(
|
||||
catalog_id=catalog.id,
|
||||
search_query=req.name.strip(),
|
||||
priority="low",
|
||||
urls_to_crawl=urls,
|
||||
)
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
|
||||
return {
|
||||
"id": str(catalog.id),
|
||||
"name": catalog.name,
|
||||
"crawl_status": catalog.crawl_status,
|
||||
"already_exists": False,
|
||||
"urls_found": len(urls),
|
||||
"message": "Program queued for research. It will be processed during off-peak hours."
|
||||
if not urls
|
||||
else f"Program queued for research with {len(urls)} known source(s).",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/catalog/{catalog_id}/adopt")
|
||||
async def adopt_program(
|
||||
catalog_id: str,
|
||||
req: AdoptRequest,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""User adopts a catalog program — creates their personal program enrollment."""
|
||||
result = await db.execute(
|
||||
select(ProgramCatalog).where(
|
||||
ProgramCatalog.id == uuid_mod.UUID(catalog_id),
|
||||
ProgramCatalog.crawl_status == "ready",
|
||||
)
|
||||
)
|
||||
catalog = result.scalar_one_or_none()
|
||||
if not catalog:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Program not found or not ready yet"
|
||||
)
|
||||
|
||||
# Check if user already has this program active
|
||||
result = await db.execute(
|
||||
select(Program).where(
|
||||
Program.user_id == user.id,
|
||||
Program.catalog_id == uuid_mod.UUID(catalog_id),
|
||||
Program.status == "active",
|
||||
)
|
||||
)
|
||||
if result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="You already have this program active")
|
||||
|
||||
# Calculate end date
|
||||
duration_weeks = catalog.duration_weeks or 12
|
||||
end_date = req.start_date + timedelta(weeks=duration_weeks)
|
||||
|
||||
program = Program(
|
||||
user_id=user.id,
|
||||
name=catalog.name,
|
||||
source="catalog",
|
||||
source_url=catalog.source_url,
|
||||
start_date=req.start_date,
|
||||
end_date=end_date,
|
||||
status="active",
|
||||
catalog_id=catalog.id,
|
||||
current_week=1,
|
||||
current_day=1,
|
||||
)
|
||||
db.add(program)
|
||||
await db.flush()
|
||||
|
||||
return {
|
||||
"id": str(program.id),
|
||||
"name": program.name,
|
||||
"start_date": program.start_date.isoformat(),
|
||||
"end_date": program.end_date.isoformat(),
|
||||
"catalog_id": str(catalog.id),
|
||||
}
|
||||
|
||||
|
||||
# ── Program Tracking Endpoints ─────────────────────────────────────────────
|
||||
|
||||
@router.get("/{program_id}/schedule")
|
||||
async def get_program_schedule(
|
||||
program_id: str,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Get the user's program schedule — today's workout, upcoming, and completed."""
|
||||
result = await db.execute(
|
||||
select(Program).where(
|
||||
Program.id == uuid_mod.UUID(program_id),
|
||||
Program.user_id == user.id,
|
||||
)
|
||||
)
|
||||
program = result.scalar_one_or_none()
|
||||
if not program or not program.catalog_id:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
# Get all catalog workouts
|
||||
result = await db.execute(
|
||||
select(CatalogWorkout)
|
||||
.where(CatalogWorkout.catalog_id == program.catalog_id)
|
||||
.order_by(CatalogWorkout.week_number, CatalogWorkout.day_number)
|
||||
)
|
||||
workouts = result.scalars().all()
|
||||
|
||||
# Get completed workout IDs
|
||||
result = await db.execute(
|
||||
select(WorkoutLog.catalog_workout_id).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
completed_ids = {row[0] for row in result.all()}
|
||||
|
||||
# Calculate current position based on start date
|
||||
today = date.today()
|
||||
days_elapsed = (today - program.start_date).days
|
||||
current_week = (days_elapsed // 7) + 1
|
||||
current_day_of_week = (days_elapsed % 7) + 1 # 1=Mon, 7=Sun
|
||||
|
||||
schedule = []
|
||||
today_workout = None
|
||||
|
||||
for w in workouts:
|
||||
entry = {
|
||||
"id": str(w.id),
|
||||
"week_number": w.week_number,
|
||||
"day_number": w.day_number,
|
||||
"workout_name": w.workout_name,
|
||||
"exercises": w.exercises,
|
||||
"rest_day": w.rest_day,
|
||||
"notes": w.notes,
|
||||
"completed": w.id in completed_ids,
|
||||
}
|
||||
schedule.append(entry)
|
||||
|
||||
if w.week_number == current_week and w.day_number == current_day_of_week:
|
||||
today_workout = entry
|
||||
|
||||
# Get catalog details for progression rules
|
||||
result = await db.execute(
|
||||
select(ProgramCatalog).where(ProgramCatalog.id == program.catalog_id)
|
||||
)
|
||||
catalog = result.scalar_one_or_none()
|
||||
|
||||
return {
|
||||
"program_id": str(program.id),
|
||||
"program_name": program.name,
|
||||
"start_date": program.start_date.isoformat(),
|
||||
"end_date": program.end_date.isoformat(),
|
||||
"current_week": current_week,
|
||||
"current_day_of_week": current_day_of_week,
|
||||
"today_workout": today_workout,
|
||||
"schedule": schedule,
|
||||
"total_workouts": len([w for w in workouts if not w.rest_day]),
|
||||
"completed_workouts": len(completed_ids),
|
||||
"progression_rules": catalog.progression_rules if catalog else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{program_id}/workout/{workout_id}")
|
||||
async def get_workout_detail(
|
||||
program_id: str,
|
||||
workout_id: str,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Get a specific workout's full exercise details."""
|
||||
# Verify program belongs to user
|
||||
result = await db.execute(
|
||||
select(Program).where(
|
||||
Program.id == uuid_mod.UUID(program_id),
|
||||
Program.user_id == user.id,
|
||||
)
|
||||
)
|
||||
program = result.scalar_one_or_none()
|
||||
if not program:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
result = await db.execute(
|
||||
select(CatalogWorkout).where(
|
||||
CatalogWorkout.id == uuid_mod.UUID(workout_id),
|
||||
CatalogWorkout.catalog_id == program.catalog_id,
|
||||
)
|
||||
)
|
||||
workout = result.scalar_one_or_none()
|
||||
if not workout:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
# Check if already completed
|
||||
result = await db.execute(
|
||||
select(WorkoutLog).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.catalog_workout_id == workout.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
log = result.scalar_one_or_none()
|
||||
|
||||
return {
|
||||
"id": str(workout.id),
|
||||
"week_number": workout.week_number,
|
||||
"day_number": workout.day_number,
|
||||
"workout_name": workout.workout_name,
|
||||
"exercises": workout.exercises,
|
||||
"rest_day": workout.rest_day,
|
||||
"notes": workout.notes,
|
||||
"completed": log is not None,
|
||||
"completed_at": log.completed_at.isoformat() if log else None,
|
||||
"exercises_completed": log.exercises_completed if log else None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{program_id}/workout/{workout_id}/complete")
|
||||
async def complete_workout(
|
||||
program_id: str,
|
||||
workout_id: str,
|
||||
req: CompleteWorkoutRequest,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Mark a program workout as complete. Awards bonus points."""
|
||||
# Verify program
|
||||
result = await db.execute(
|
||||
select(Program).where(
|
||||
Program.id == uuid_mod.UUID(program_id),
|
||||
Program.user_id == user.id,
|
||||
Program.status == "active",
|
||||
)
|
||||
)
|
||||
program = result.scalar_one_or_none()
|
||||
if not program:
|
||||
raise HTTPException(status_code=404, detail="Active program not found")
|
||||
|
||||
# Verify workout belongs to program's catalog
|
||||
result = await db.execute(
|
||||
select(CatalogWorkout).where(
|
||||
CatalogWorkout.id == uuid_mod.UUID(workout_id),
|
||||
CatalogWorkout.catalog_id == program.catalog_id,
|
||||
)
|
||||
)
|
||||
workout = result.scalar_one_or_none()
|
||||
if not workout:
|
||||
raise HTTPException(status_code=404, detail="Workout not found")
|
||||
|
||||
# Check not already completed
|
||||
result = await db.execute(
|
||||
select(WorkoutLog).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.catalog_workout_id == workout.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
if result.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="Workout already completed")
|
||||
|
||||
# Award points — program workouts get 75 pts (vs 50 for generic)
|
||||
PROGRAM_WORKOUT_POINTS = 75
|
||||
|
||||
# Log the workout completion
|
||||
log = WorkoutLog(
|
||||
user_id=user.id,
|
||||
program_id=program.id,
|
||||
catalog_workout_id=workout.id,
|
||||
exercises_completed=req.exercises_completed,
|
||||
points_earned=PROGRAM_WORKOUT_POINTS,
|
||||
notes=req.notes,
|
||||
)
|
||||
db.add(log)
|
||||
|
||||
# Also log as activity for the points engine
|
||||
activity = await log_activity_with_points(
|
||||
db=db,
|
||||
user_id=user.id,
|
||||
category="workout",
|
||||
activity_date=date.today(),
|
||||
title=f"{program.name}: {workout.workout_name or f'Week {workout.week_number} Day {workout.day_number}'}",
|
||||
description=f"Completed program workout ({len(workout.exercises)} exercises)",
|
||||
source="program",
|
||||
program_id=program.id,
|
||||
program_day=workout.day_number,
|
||||
metadata={"catalog_workout_id": str(workout.id), "program_points": True},
|
||||
)
|
||||
|
||||
# Override points with program bonus if higher
|
||||
if activity.points_earned < PROGRAM_WORKOUT_POINTS:
|
||||
activity.points_earned = PROGRAM_WORKOUT_POINTS
|
||||
|
||||
await db.flush()
|
||||
|
||||
# Check for weekly bonus (all workouts in current week completed)
|
||||
weekly_bonus = await check_weekly_bonus(db, user, program, workout.week_number)
|
||||
|
||||
# Check for program completion bonus
|
||||
completion_bonus = await check_completion_bonus(db, user, program)
|
||||
|
||||
total_points = PROGRAM_WORKOUT_POINTS + weekly_bonus + completion_bonus
|
||||
|
||||
return {
|
||||
"workout_log_id": str(log.id),
|
||||
"points_earned": PROGRAM_WORKOUT_POINTS,
|
||||
"weekly_bonus": weekly_bonus,
|
||||
"completion_bonus": completion_bonus,
|
||||
"total_points": total_points,
|
||||
"workout_name": workout.workout_name,
|
||||
}
|
||||
|
||||
|
||||
async def check_weekly_bonus(
|
||||
db: AsyncSession, user: User, program: Program, week_number: int
|
||||
) -> int:
|
||||
"""Check if all workouts for a week are completed. Returns bonus points."""
|
||||
WEEKLY_BONUS = 50
|
||||
|
||||
# Count total non-rest workouts in this week
|
||||
result = await db.execute(
|
||||
select(func.count(CatalogWorkout.id)).where(
|
||||
CatalogWorkout.catalog_id == program.catalog_id,
|
||||
CatalogWorkout.week_number == week_number,
|
||||
CatalogWorkout.rest_day == False,
|
||||
)
|
||||
)
|
||||
total_in_week = result.scalar() or 0
|
||||
if total_in_week == 0:
|
||||
return 0
|
||||
|
||||
# Count completed in this week
|
||||
result = await db.execute(
|
||||
select(func.count(WorkoutLog.id))
|
||||
.join(CatalogWorkout, WorkoutLog.catalog_workout_id == CatalogWorkout.id)
|
||||
.where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
CatalogWorkout.week_number == week_number,
|
||||
)
|
||||
)
|
||||
completed_in_week = result.scalar() or 0
|
||||
|
||||
if completed_in_week >= total_in_week:
|
||||
# Award weekly bonus as an activity
|
||||
await log_activity_with_points(
|
||||
db=db,
|
||||
user_id=user.id,
|
||||
category="bonus",
|
||||
activity_date=date.today(),
|
||||
title=f"{program.name}: Week {week_number} Complete!",
|
||||
source="program",
|
||||
program_id=program.id,
|
||||
metadata={"weekly_bonus": True, "week": week_number},
|
||||
)
|
||||
return WEEKLY_BONUS
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
async def check_completion_bonus(
|
||||
db: AsyncSession, user: User, program: Program
|
||||
) -> int:
|
||||
"""Check if all program workouts are completed. Returns bonus points."""
|
||||
COMPLETION_BONUS = 200
|
||||
|
||||
# Count total non-rest workouts in program
|
||||
result = await db.execute(
|
||||
select(func.count(CatalogWorkout.id)).where(
|
||||
CatalogWorkout.catalog_id == program.catalog_id,
|
||||
CatalogWorkout.rest_day == False,
|
||||
)
|
||||
)
|
||||
total = result.scalar() or 0
|
||||
|
||||
# Count completed
|
||||
result = await db.execute(
|
||||
select(func.count(WorkoutLog.id)).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
completed = result.scalar() or 0
|
||||
|
||||
if completed >= total and total > 0:
|
||||
program.status = "completed"
|
||||
await log_activity_with_points(
|
||||
db=db,
|
||||
user_id=user.id,
|
||||
category="bonus",
|
||||
activity_date=date.today(),
|
||||
title=f"{program.name}: Program Complete!",
|
||||
source="program",
|
||||
program_id=program.id,
|
||||
metadata={"completion_bonus": True},
|
||||
)
|
||||
return COMPLETION_BONUS
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@router.get("/{program_id}/progress")
|
||||
async def get_program_progress(
|
||||
program_id: str,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
):
|
||||
"""Program progress overview."""
|
||||
result = await db.execute(
|
||||
select(Program).where(
|
||||
Program.id == uuid_mod.UUID(program_id),
|
||||
Program.user_id == user.id,
|
||||
)
|
||||
)
|
||||
program = result.scalar_one_or_none()
|
||||
if not program or not program.catalog_id:
|
||||
raise HTTPException(status_code=404)
|
||||
|
||||
# Total workouts
|
||||
result = await db.execute(
|
||||
select(func.count(CatalogWorkout.id)).where(
|
||||
CatalogWorkout.catalog_id == program.catalog_id,
|
||||
CatalogWorkout.rest_day == False,
|
||||
)
|
||||
)
|
||||
total = result.scalar() or 0
|
||||
|
||||
# Completed
|
||||
result = await db.execute(
|
||||
select(func.count(WorkoutLog.id)).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
completed = result.scalar() or 0
|
||||
|
||||
# Total points from this program
|
||||
result = await db.execute(
|
||||
select(func.coalesce(func.sum(WorkoutLog.points_earned), 0)).where(
|
||||
WorkoutLog.program_id == program.id,
|
||||
WorkoutLog.user_id == user.id,
|
||||
)
|
||||
)
|
||||
total_points = result.scalar() or 0
|
||||
|
||||
today = date.today()
|
||||
days_elapsed = (today - program.start_date).days
|
||||
current_week = (days_elapsed // 7) + 1
|
||||
|
||||
return {
|
||||
"program_id": str(program.id),
|
||||
"name": program.name,
|
||||
"status": program.status,
|
||||
"start_date": program.start_date.isoformat(),
|
||||
"end_date": program.end_date.isoformat(),
|
||||
"current_week": current_week,
|
||||
"total_workouts": total,
|
||||
"completed_workouts": completed,
|
||||
"completion_pct": round((completed / total * 100) if total > 0 else 0, 1),
|
||||
"total_points_earned": total_points,
|
||||
}
|
||||
90
backend/app/services/crawl_scheduler.py
Normal file
90
backend/app/services/crawl_scheduler.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
"""Crawl queue scheduler — processes jobs respecting priority and time constraints."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import async_session
|
||||
from app.models.catalog import CrawlQueue
|
||||
from app.services.program_research import process_crawl_job
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ICT = UTC+7. Off-peak: 22:00-06:00 ICT = 15:00-23:00 UTC
|
||||
OFFPEAK_START_UTC = 15 # 22:00 ICT
|
||||
OFFPEAK_END_UTC = 23 # 06:00 ICT
|
||||
|
||||
|
||||
def is_offpeak() -> bool:
|
||||
"""Check if current time is off-peak (22:00-06:00 ICT)."""
|
||||
hour = datetime.now(timezone.utc).hour
|
||||
return hour >= OFFPEAK_START_UTC or hour < OFFPEAK_END_UTC
|
||||
|
||||
|
||||
def is_weekend() -> bool:
|
||||
"""Check if today is Saturday (5) or Sunday (6)."""
|
||||
return datetime.now(timezone.utc).weekday() >= 5
|
||||
|
||||
|
||||
async def has_higher_priority_jobs(db: AsyncSession) -> bool:
|
||||
"""Check if any HIGH or MEDIUM priority jobs are pending."""
|
||||
result = await db.execute(
|
||||
select(func.count(CrawlQueue.id)).where(
|
||||
CrawlQueue.priority.in_(["high", "medium"]),
|
||||
CrawlQueue.status == "pending",
|
||||
)
|
||||
)
|
||||
return (result.scalar() or 0) > 0
|
||||
|
||||
|
||||
async def get_next_pending_job(db: AsyncSession) -> CrawlQueue | None:
|
||||
"""Get the oldest pending LOW priority crawl job."""
|
||||
result = await db.execute(
|
||||
select(CrawlQueue)
|
||||
.where(CrawlQueue.status == "pending", CrawlQueue.priority == "low")
|
||||
.order_by(CrawlQueue.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def run_queue_cycle() -> None:
|
||||
"""Process one job from the queue if conditions are met."""
|
||||
# Only run during off-peak or weekends
|
||||
if not is_offpeak() and not is_weekend():
|
||||
return
|
||||
|
||||
async with async_session() as db:
|
||||
try:
|
||||
# Defer to higher priority work
|
||||
if await has_higher_priority_jobs(db):
|
||||
logger.debug("Higher priority jobs pending — skipping fitness crawl")
|
||||
return
|
||||
|
||||
job = await get_next_pending_job(db)
|
||||
if not job:
|
||||
return
|
||||
|
||||
logger.info(f"Processing crawl job {job.id}: {job.search_query}")
|
||||
await process_crawl_job(db, job)
|
||||
await db.commit()
|
||||
logger.info(f"Crawl job {job.id} completed with status: {job.status}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Crawl queue cycle error: {e}")
|
||||
await db.rollback()
|
||||
|
||||
|
||||
async def crawl_queue_loop() -> None:
|
||||
"""Background loop — checks queue every 5 minutes."""
|
||||
logger.info("Crawl queue scheduler started")
|
||||
while True:
|
||||
try:
|
||||
await run_queue_cycle()
|
||||
except Exception as e:
|
||||
logger.error(f"Crawl queue loop error: {e}")
|
||||
await asyncio.sleep(300) # 5 minutes
|
||||
262
backend/app/services/program_research.py
Normal file
262
backend/app/services/program_research.py
Normal file
|
|
@ -0,0 +1,262 @@
|
|||
"""Program research service — crawl fitness programs and extract structured data via Groq."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.catalog import ProgramCatalog, CatalogWorkout, CrawlQueue
|
||||
from app.config import settings
|
||||
|
||||
|
||||
# ── Known program sources (skip search for these) ──────────────────────────
|
||||
|
||||
KNOWN_SOURCES: dict[str, list[str]] = {
|
||||
"stronglifts 5x5": ["https://stronglifts.com/5x5/"],
|
||||
"stronglifts": ["https://stronglifts.com/5x5/"],
|
||||
"starting strength": ["https://startingstrength.com/get-started/programs"],
|
||||
"couch to 5k": [
|
||||
"https://www.nhs.uk/live-well/exercise/running-and-aerobic-exercises/get-running-with-couch-to-5k/"
|
||||
],
|
||||
"c25k": [
|
||||
"https://www.nhs.uk/live-well/exercise/running-and-aerobic-exercises/get-running-with-couch-to-5k/"
|
||||
],
|
||||
"greyskull lp": ["https://physiqz.com/powerlifting-programs/greyskull-lp-best-powerlifting-routine-for-beginners/"],
|
||||
"5/3/1": ["https://www.jimwendler.com/blogs/jimwendler-com/101065094-5-3-1-for-a-beginner"],
|
||||
"wendler 531": ["https://www.jimwendler.com/blogs/jimwendler-com/101065094-5-3-1-for-a-beginner"],
|
||||
"push pull legs": ["https://www.muscleandstrength.com/workouts/6-day-push-pull-legs-planet-fitness-workout"],
|
||||
"ppl": ["https://www.muscleandstrength.com/workouts/6-day-push-pull-legs-planet-fitness-workout"],
|
||||
}
|
||||
|
||||
|
||||
# ── Groq extraction prompt ─────────────────────────────────────────────────
|
||||
|
||||
EXTRACTION_PROMPT = """You are a fitness program parser. Extract the workout program from the content below into structured JSON.
|
||||
|
||||
Return ONLY valid JSON matching this exact schema — no markdown fences, no explanation:
|
||||
|
||||
{
|
||||
"name": "Program Name",
|
||||
"description": "Brief 1-2 sentence description",
|
||||
"duration_weeks": 12,
|
||||
"frequency_per_week": 3,
|
||||
"difficulty": "beginner",
|
||||
"category": "strength",
|
||||
"equipment": ["barbell", "squat rack", "bench"],
|
||||
"progression_rules": "Human-readable progression description. E.g. 'Add 2.5kg per session on all lifts. If you fail a weight 3 sessions in a row, deload 10%.'",
|
||||
"workouts": [
|
||||
{
|
||||
"week": 1,
|
||||
"day": 1,
|
||||
"name": "Workout A",
|
||||
"rest_day": false,
|
||||
"exercises": [
|
||||
{
|
||||
"name": "Squat",
|
||||
"sets": 5,
|
||||
"reps": 5,
|
||||
"weight_instruction": "Start with empty bar (20kg), add 2.5kg each session",
|
||||
"rest_seconds": 180,
|
||||
"notes": "Below parallel"
|
||||
}
|
||||
],
|
||||
"notes": null
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Rules:
|
||||
- difficulty must be one of: beginner, intermediate, advanced
|
||||
- category must be one of: strength, cardio, flexibility, hybrid
|
||||
- For alternating programs (A/B/A rotation), output the minimum unique weeks needed to show the pattern. Explain the rotation in progression_rules.
|
||||
- If a day is a rest day, set rest_day: true and exercises: []
|
||||
- weight_instruction should be specific if the source provides guidance, otherwise "Use appropriate weight"
|
||||
- rest_seconds: use the source value, or 60-90 for hypertrophy, 180-300 for strength, 30-60 for cardio
|
||||
- Include ALL exercises mentioned for each workout day
|
||||
"""
|
||||
|
||||
|
||||
def slugify(name: str) -> str:
|
||||
"""Convert program name to URL-safe slug."""
|
||||
s = name.lower().strip()
|
||||
s = re.sub(r"[^a-z0-9\s-]", "", s)
|
||||
s = re.sub(r"[\s-]+", "-", s)
|
||||
return s[:200]
|
||||
|
||||
|
||||
def find_urls_for_program(query: str) -> list[str]:
|
||||
"""Look up known URLs or return empty list for unknown programs."""
|
||||
q = query.lower().strip()
|
||||
for key, urls in KNOWN_SOURCES.items():
|
||||
if key in q or q in key:
|
||||
return urls
|
||||
return []
|
||||
|
||||
|
||||
async def crawl_url(url: str) -> str:
|
||||
"""Call ttp-crawler on subo to fetch a URL as markdown."""
|
||||
proc = subprocess.run(
|
||||
[
|
||||
"python3", "/home/claude/ttp-crawler/crawl.py",
|
||||
"fetch", url, "--output", "md",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
cwd="/home/claude/ttp-crawler",
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"Crawl failed: {proc.stderr[:500]}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
async def extract_with_groq(crawled_content: str) -> dict[str, Any]:
|
||||
"""Send crawled markdown to Groq for structured extraction."""
|
||||
# Truncate to fit context — Llama 3.3 70B has 128k context but we keep it reasonable
|
||||
content = crawled_content[:15000]
|
||||
|
||||
async with httpx.AsyncClient(timeout=90.0) as client:
|
||||
resp = await client.post(
|
||||
"https://api.groq.com/openai/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {settings.GROQ_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": "llama-3.3-70b-versatile",
|
||||
"messages": [
|
||||
{"role": "system", "content": EXTRACTION_PROMPT},
|
||||
{"role": "user", "content": content},
|
||||
],
|
||||
"response_format": {"type": "json_object"},
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 8000,
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
raw = data["choices"][0]["message"]["content"]
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def validate_extraction(data: dict) -> tuple[bool, str]:
|
||||
"""Basic validation of Groq extraction output."""
|
||||
required = ["name", "workouts"]
|
||||
for field in required:
|
||||
if field not in data:
|
||||
return False, f"Missing required field: {field}"
|
||||
|
||||
if not isinstance(data.get("workouts"), list) or len(data["workouts"]) == 0:
|
||||
return False, "No workouts extracted"
|
||||
|
||||
for i, w in enumerate(data["workouts"]):
|
||||
if not w.get("rest_day", False) and not w.get("exercises"):
|
||||
return False, f"Workout {i + 1} has no exercises and is not a rest day"
|
||||
|
||||
return True, "OK"
|
||||
|
||||
|
||||
async def populate_catalog_from_extraction(
|
||||
db: AsyncSession, catalog: ProgramCatalog, data: dict
|
||||
) -> None:
|
||||
"""Write validated extraction data into catalog + catalog_workouts."""
|
||||
catalog.name = data.get("name", catalog.name)
|
||||
catalog.description = data.get("description")
|
||||
catalog.duration_weeks = data.get("duration_weeks")
|
||||
catalog.frequency_per_week = data.get("frequency_per_week")
|
||||
catalog.equipment = data.get("equipment", [])
|
||||
catalog.difficulty = data.get("difficulty")
|
||||
catalog.category = data.get("category")
|
||||
catalog.progression_rules = data.get("progression_rules")
|
||||
catalog.structured_data = data
|
||||
catalog.crawl_status = "ready"
|
||||
catalog.crawled_at = datetime.now(timezone.utc)
|
||||
|
||||
for w in data.get("workouts", []):
|
||||
workout = CatalogWorkout(
|
||||
catalog_id=catalog.id,
|
||||
week_number=w.get("week", 1),
|
||||
day_number=w.get("day", 1),
|
||||
workout_name=w.get("name"),
|
||||
exercises=w.get("exercises", []),
|
||||
notes=w.get("notes"),
|
||||
rest_day=w.get("rest_day", False),
|
||||
)
|
||||
db.add(workout)
|
||||
|
||||
await db.flush()
|
||||
|
||||
|
||||
async def process_crawl_job(db: AsyncSession, job: CrawlQueue) -> None:
|
||||
"""Execute a single crawl queue job end-to-end."""
|
||||
job.status = "running"
|
||||
job.started_at = datetime.now(timezone.utc)
|
||||
await db.flush()
|
||||
|
||||
catalog = None
|
||||
if job.catalog_id:
|
||||
result = await db.execute(
|
||||
select(ProgramCatalog).where(ProgramCatalog.id == job.catalog_id)
|
||||
)
|
||||
catalog = result.scalar_one_or_none()
|
||||
|
||||
try:
|
||||
# Step 1: Determine URLs to crawl
|
||||
urls = job.urls_to_crawl if job.urls_to_crawl else find_urls_for_program(job.search_query)
|
||||
if not urls:
|
||||
raise ValueError(
|
||||
f"No known sources for '{job.search_query}'. "
|
||||
"Add URLs manually or extend KNOWN_SOURCES."
|
||||
)
|
||||
|
||||
# Step 2: Crawl
|
||||
crawled_parts = []
|
||||
for url in urls:
|
||||
try:
|
||||
content = await crawl_url(url)
|
||||
if len(content.strip()) > 200: # skip near-empty crawls
|
||||
crawled_parts.append(content)
|
||||
except Exception as e:
|
||||
crawled_parts.append(f"[Crawl error for {url}: {e}]")
|
||||
|
||||
if not crawled_parts:
|
||||
raise ValueError("All crawl attempts returned empty content")
|
||||
|
||||
combined = "\n\n---\n\n".join(crawled_parts)
|
||||
job.crawled_content = combined[:50000] # store for debugging, cap at 50KB
|
||||
|
||||
# Step 3: Extract via Groq
|
||||
if catalog:
|
||||
catalog.crawl_status = "extracting"
|
||||
await db.flush()
|
||||
|
||||
extracted = await extract_with_groq(combined)
|
||||
|
||||
# Step 4: Validate
|
||||
valid, reason = validate_extraction(extracted)
|
||||
if not valid:
|
||||
raise ValueError(f"Extraction validation failed: {reason}")
|
||||
|
||||
# Step 5: Populate catalog
|
||||
if catalog:
|
||||
await populate_catalog_from_extraction(db, catalog, extracted)
|
||||
|
||||
job.status = "done"
|
||||
job.completed_at = datetime.now(timezone.utc)
|
||||
|
||||
except Exception as e:
|
||||
job.status = "failed"
|
||||
job.error = str(e)[:2000]
|
||||
job.completed_at = datetime.now(timezone.utc)
|
||||
if catalog:
|
||||
catalog.crawl_status = "failed"
|
||||
catalog.crawl_error = str(e)[:2000]
|
||||
|
||||
await db.flush()
|
||||
Loading…
Add table
Add a link
Reference in a new issue