deploy: ba672bd from MK23IS092/msrit_clockwork

.env.example

@@ -0,0 +1,42 @@
+# Core
+API_HOST=0.0.0.0
+API_PORT=8000
+
+# Model providers
+GEMINI_API_KEY=
+HUGGINGFACE_TOKEN=
+KAGGLE_USERNAME=
+KAGGLE_KEY=
+GITHUB_TOKEN=
+TELEGRAM_BOT_TOKEN=
+# Optional if unset: after you message the bot once, restart the API or run scripts/discover_telegram_chat.py
+TELEGRAM_CHAT_ID=
+
+# Ingestion controls
+ENABLE_PATENTS_REAL=true
+ENABLE_STARTUPS_REAL=true
+ENABLE_SOCIAL_REAL=true
+ENABLE_BLOG_REAL=true
+ALLOW_SIMULATED_SOURCES=true
+
+# Distributed infra backends
+DB_BACKEND=postgres
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5432
+POSTGRES_DB=vectormind
+POSTGRES_USER=vectormind
+POSTGRES_PASSWORD=vectormind
+POSTGRES_DSN=postgresql://vectormind:vectormind@localhost:5432/vectormind
+
+STATE_STORE_BACKEND=redis
+REDIS_URL=redis://localhost:6379/0
+
+MESSAGE_BUS_BACKEND=kafka_mirror
+KAFKA_BOOTSTRAP_SERVERS=localhost:9092
+KAFKA_TOPIC_PREFIX=vectormind
+
+# Pipeline runtime
+PIPELINE_RUN_TIMEOUT_SECONDS=1800
+PIPELINE_MAX_CONCURRENT_RUNS=2
+PIPELINE_MAX_RETRIES=1
+PIPELINE_RETRY_BACKOFF_SECONDS=5

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,64 @@
+# ─────────────────────────────────────────────────────────────────────────────
+# VectorMind backend — Hugging Face Spaces (Docker SDK) image.
+#
+# Why this Dockerfile is the way it is:
+#   • HF Spaces (Docker SDK) expect the app to listen on port 7860.
+#   • HF builds are slow; we install CPU-only torch from the PyTorch CPU index
+#     so we don't pull a multi-GB CUDA wheel that we'll never use.
+#   • Persistence: HF Spaces give us /data (writable) but the rest of the FS
+#     resets on rebuild. We point DATA_DIR there so SQLite + run logs survive
+#     restarts even though they're wiped on space restart-with-rebuild.
+#   • Everything heavy is opt-in via env vars (Postgres/Redis/Kafka). For the
+#     demo the SQLite + in-memory paths are used — no external services
+#     required to stand up the demo.
+# ─────────────────────────────────────────────────────────────────────────────
+
+FROM python:3.11-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    HF_HOME=/data/.cache/huggingface \
+    TRANSFORMERS_CACHE=/data/.cache/huggingface \
+    SENTENCE_TRANSFORMERS_HOME=/data/.cache/huggingface
+
+# System deps:
+#  • libgomp1 — required by torch (OpenMP runtime)
+#  • git/curl — handy for runtime downloads of model weights
+#  • build-essential not needed because we install slim binary wheels
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+        git curl libgomp1 ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install CPU-only torch first so the heavy wheel is cached and we don't
+# accidentally pull a 2GB CUDA build via a transitive dep.
+RUN pip install --upgrade pip \
+ && pip install torch==2.4.0 --index-url https://download.pytorch.org/whl/cpu
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY . /app
+
+# HF Spaces conventions:
+#   • port 7860
+#   • /data is the only persistent writable mount
+ENV API_HOST=0.0.0.0 \
+    API_PORT=7860 \
+    DB_BACKEND=sqlite \
+    STATE_STORE_BACKEND=sqlite \
+    MESSAGE_BUS_BACKEND=in_memory \
+    USE_MOCK_LLM=false \
+    HF_DEPLOYMENT=true
+
+# /data is writable on HF; pre-create our subfolders so first-write succeeds.
+RUN mkdir -p /data/cache /data/pipeline_runs \
+ && chmod -R 777 /data
+
+EXPOSE 7860
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,16 @@
 ---
-title: Zamzung
-emoji: 🏃
-colorFrom: purple
-colorTo: gray
+title: VectorMind Backend
+emoji: 🧠
+colorFrom: indigo
+colorTo: purple
 sdk: docker
+app_port: 7860
 pinned: false
-license: apache-2.0
+short_description: Autonomous AI research intelligence backend
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VectorMind Backend
+
+FastAPI service that powers the VectorMind Android app. Source of
+truth lives in [the GitHub repo](https://github.com/) — this Space
+is auto-deployed on every push to `main`.

agents/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Agents Package

agents/base_agent.py

@@ -0,0 +1,159 @@
+"""Base Agent — Abstract base class for all OpenClaw agents.
+
+Provides lifecycle management, event loop, health checking,
+and state checkpointing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Optional
+
+import config
+from agents.message_bus import MessageBus
+from ingestion.schema import AgentEvent
+
+logger = logging.getLogger("vectormind.agent")
+
+try:
+    import redis.asyncio as redis_async
+except Exception:  # pragma: no cover - optional dependency
+    redis_async = None
+
+
+class BaseAgent(ABC):
+    """Abstract base class for VectorMind agents."""
+
+    def __init__(self, name: str):
+        self.name = name
+        self.bus = MessageBus.get_instance()
+        self._running = False
+        self._task: Optional[asyncio.Task] = None
+        self._last_heartbeat = datetime.utcnow()
+        self._events_processed = 0
+        self._status = "idle"
+        self._subscribed_topics: list[str] = []
+        self._queues: list[asyncio.Queue] = []
+        self._state_store = None
+
+    @property
+    def status(self) -> str:
+        return self._status
+
+    @property
+    def is_running(self) -> bool:
+        return self._running
+
+    def get_health(self) -> dict:
+        """Return agent health status."""
+        return {
+            "name": self.name,
+            "status": self._status,
+            "running": self._running,
+            "events_processed": self._events_processed,
+            "last_heartbeat": self._last_heartbeat.isoformat(),
+        }
+
+    def subscribe(self, topic: str):
+        """Subscribe to a message bus topic."""
+        queue = self.bus.subscribe(topic)
+        self._queues.append(queue)
+        self._subscribed_topics.append(topic)
+        logger.info(f"Agent '{self.name}' subscribed to '{topic}'")
+
+    async def publish(self, topic: str, payload: dict):
+        """Publish an event to the message bus."""
+        await self.bus.publish_simple(topic, self.name, payload)
+
+    async def start(self):
+        """Start the agent's event loop."""
+        self._running = True
+        self._status = "running"
+        if config.STATE_STORE_BACKEND == "redis" and redis_async is not None:
+            self._state_store = redis_async.from_url(config.REDIS_URL, decode_responses=True)
+        self.setup()
+        logger.info(f"Agent '{self.name}' started")
+        self._task = asyncio.create_task(self._run_loop())
+
+    async def stop(self):
+        """Stop the agent."""
+        self._running = False
+        self._status = "stopped"
+        if self._task:
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+        if self._state_store is not None:
+            await self._state_store.close()
+        logger.info(f"Agent '{self.name}' stopped")
+
+    async def _run_loop(self):
+        """Main event processing loop."""
+        while self._running:
+            try:
+                # Process events from all subscribed queues
+                for queue in self._queues:
+                    try:
+                        event = queue.get_nowait()
+                        await self.process_event(event)
+                        self._events_processed += 1
+                        self._last_heartbeat = datetime.utcnow()
+                        await self._checkpoint_state()
+                    except asyncio.QueueEmpty:
+                        continue
+
+                # Run periodic tasks
+                await self.periodic_task()
+
+                # Small sleep to prevent busy-waiting
+                await asyncio.sleep(0.1)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Agent '{self.name}' error: {e}")
+                self._status = "error"
+                await asyncio.sleep(1.0)
+                self._status = "running"
+
+    async def _checkpoint_state(self):
+        """Persist light agent heartbeat/status to Redis when enabled."""
+        if self._state_store is None:
+            return
+        try:
+            key = f"vectormind:agent:{self.name}:state"
+            await self._state_store.hset(
+                key,
+                mapping={
+                    "status": self._status,
+                    "events_processed": str(self._events_processed),
+                    "last_heartbeat": self._last_heartbeat.isoformat(),
+                },
+            )
+            await self._state_store.expire(key, 3600)
+        except Exception as e:
+            logger.error(f"Agent '{self.name}' checkpoint failed: {e}")
+
+    def setup(self):
+        """Optional setup hook called before the event loop starts."""
+        pass
+
+    @abstractmethod
+    async def process_event(self, event: AgentEvent):
+        """Process a single event from the message bus.
+
+        Must be implemented by subclasses.
+        """
+        pass
+
+    async def periodic_task(self):
+        """Optional periodic task that runs each loop iteration.
+
+        Override in subclasses for scheduled work.
+        """
+        pass

agents/ingestion_agent.py

@@ -0,0 +1,334 @@
+"""Ingestion Agent — Manages all data source crawlers.
+
+Orchestrates arXiv and GitHub crawlers, monitors source health,
+and publishes new research signals to the message bus.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime
+
+from agents.base_agent import BaseAgent
+from embeddings.engine import EmbeddingEngine
+from embeddings.vector_store import VectorStore
+from ingestion.arxiv_crawler import ArxivCrawler
+from ingestion.blog_crawler import BlogCrawler
+from ingestion.github_crawler import GitHubCrawler
+from ingestion.patent_crawler import PatentCrawler
+from ingestion.schema import AgentEvent, ResearchSignal, SignalSource
+from ingestion.social_crawler import SocialCrawler
+from ingestion.startup_crawler import StartupCrawler
+import config
+
+logger = logging.getLogger("vectormind.ingestion_agent")
+
+
+class IngestionAgent(BaseAgent):
+    """Agent that manages data ingestion from all sources."""
+
+    def __init__(self):
+        super().__init__("IngestionAgent")
+        self.arxiv_crawler = ArxivCrawler(
+            categories=config.ARXIV_CATEGORIES,
+            max_results=config.ARXIV_MAX_RESULTS,
+        )
+        self.github_crawler = GitHubCrawler(
+            languages=config.GITHUB_TRENDING_LANGUAGES,
+            max_results=config.GITHUB_MAX_RESULTS,
+            token=config.GITHUB_TOKEN,
+        )
+        self.patent_crawler = PatentCrawler()
+        self.startup_crawler = StartupCrawler()
+        self.social_crawler = SocialCrawler()
+        self.blog_crawler = BlogCrawler()
+        self.embedding_engine = EmbeddingEngine.get_instance()
+        self.vector_store = VectorStore.get_instance()
+
+        self._last_ingestion = None
+        self._ingestion_count = 0
+        self._source_health = {
+            "arxiv": {"status": "healthy", "last_success": None, "failures": 0},
+            "github": {"status": "healthy", "last_success": None, "failures": 0},
+            "patents": {"status": "healthy", "last_success": None, "failures": 0},
+            "startups": {"status": "healthy", "last_success": None, "failures": 0},
+            "social": {"status": "healthy", "last_success": None, "failures": 0},
+            "blog": {"status": "healthy", "last_success": None, "failures": 0},
+        }
+
+    def setup(self):
+        self.subscribe("ingestion.trigger")
+
+    async def process_event(self, event: AgentEvent):
+        """Handle ingestion trigger events."""
+        if event.topic == "ingestion.trigger":
+            category = event.payload.get("category")
+            source = event.payload.get("source", "all")
+            await self.run_ingestion(source=source, category=category)
+
+    async def run_ingestion(
+        self,
+        source: str = "all",
+        category: str = None,
+    ) -> list[ResearchSignal]:
+        """Run a full ingestion cycle.
+
+        Args:
+            source: 'arxiv', 'github', or 'all'
+            category: Optional arXiv category filter
+
+        Returns:
+            List of new research signals ingested
+        """
+        self._status = "ingesting"
+        all_signals = []
+
+        # Fetch from arXiv
+        if source in ("all", "arxiv"):
+            try:
+                arxiv_signals = await self.arxiv_crawler.fetch_recent_papers(
+                    category=category
+                )
+                all_signals.extend(arxiv_signals)
+                self._source_health["arxiv"]["status"] = "healthy"
+                self._source_health["arxiv"]["last_success"] = (
+                    datetime.utcnow().isoformat()
+                )
+                self._source_health["arxiv"]["failures"] = 0
+            except Exception as e:
+                logger.error(f"arXiv ingestion failed: {e}")
+                self._source_health["arxiv"]["failures"] += 1
+                if self._source_health["arxiv"]["failures"] >= 3:
+                    self._source_health["arxiv"]["status"] = "unhealthy"
+
+        # Fetch from GitHub
+        if source in ("all", "github"):
+            try:
+                for topic in ["machine-learning", "deep-learning", "transformers"]:
+                    github_signals = await self.github_crawler.fetch_trending_repos(
+                        topic=topic
+                    )
+                    all_signals.extend(github_signals)
+                self._source_health["github"]["status"] = "healthy"
+                self._source_health["github"]["last_success"] = (
+                    datetime.utcnow().isoformat()
+                )
+                self._source_health["github"]["failures"] = 0
+            except Exception as e:
+                logger.error(f"GitHub ingestion failed: {e}")
+                self._source_health["github"]["failures"] += 1
+                if self._source_health["github"]["failures"] >= 3:
+                    self._source_health["github"]["status"] = "unhealthy"
+        # Fetch from Patents
+        if source in ("all", "patents"):
+            try:
+                patent_signals = []
+                if config.ENABLE_PATENTS_REAL:
+                    patent_signals = await self.patent_crawler.fetch_recent_patents(
+                        max_results=config.PATENTS_MAX_RESULTS
+                    )
+                if not patent_signals and config.ALLOW_SIMULATED_SOURCES:
+                    patent_signals = [
+                        ResearchSignal(
+                            source=SignalSource.PATENT,
+                            source_id="US-2026-0012345",
+                            title="Distributed Multi-Agent Reasoning via Sparse Attention Meshes",
+                            authors=["VectorMind R&D"],
+                            raw_text="A method and system for optimizing multi-agent reasoning in decentralized networks...",
+                            url="https://patents.google.com/patent/US20260012345A1",
+                            metadata={
+                                "patent_number": "US20260012345",
+                                "assignee": "Samsung R&D",
+                                "simulated": True,
+                            },
+                        )
+                    ]
+                all_signals.extend(patent_signals)
+                self._source_health["patents"]["status"] = "healthy"
+                self._source_health["patents"]["last_success"] = datetime.utcnow().isoformat()
+            except Exception as e:
+                logger.error(f"Patent ingestion failed: {e}")
+                self._source_health["patents"]["status"] = "unhealthy"
+                if config.ALLOW_SIMULATED_SOURCES:
+                    all_signals.extend(self._simulated_patent_signals())
+
+        # Fetch from Startup Ecosystem
+        if source in ("all", "startups"):
+            try:
+                startup_signals = []
+                if config.ENABLE_STARTUPS_REAL:
+                    startup_signals = await self.startup_crawler.fetch_startup_signals(
+                        max_results=config.STARTUPS_MAX_RESULTS
+                    )
+                if not startup_signals and config.ALLOW_SIMULATED_SOURCES:
+                    startup_signals = [
+                        ResearchSignal(
+                            source=SignalSource.STARTUP,
+                            source_id="YC-W26-VECT",
+                            title="Seed Funding: NeuroForge AI (YC W26)",
+                            authors=["YC"],
+                            raw_text="NeuroForge AI raises $5M to commercialize sparse attention architectures.",
+                            url="https://ycombinator.com/companies/neuroforge",
+                            metadata={"funding_round": "Seed", "amount": "$5M", "simulated": True},
+                        )
+                    ]
+                all_signals.extend(startup_signals)
+                self._source_health["startups"]["status"] = "healthy"
+                self._source_health["startups"]["last_success"] = datetime.utcnow().isoformat()
+            except Exception:
+                self._source_health["startups"]["status"] = "unhealthy"
+                if config.ALLOW_SIMULATED_SOURCES:
+                    all_signals.extend(self._simulated_startup_signals())
+
+        # Fetch from Social (Hacker News)
+        if source in ("all", "social"):
+            try:
+                social_signals = []
+                if config.ENABLE_SOCIAL_REAL:
+                    social_signals = await self.social_crawler.fetch_hn_signals(
+                        max_results=config.SOCIAL_MAX_RESULTS
+                    )
+                if not social_signals and config.ALLOW_SIMULATED_SOURCES:
+                    social_signals = [
+                        ResearchSignal(
+                            source=SignalSource.SOCIAL,
+                            source_id="HN-4123456",
+                            title="Show HN: VectorMind - Open Source Research Intelligence",
+                            authors=["hn_user"],
+                            raw_text="The first agentic platform for autonomous research...",
+                            url="https://news.ycombinator.com/item?id=4123456",
+                            metadata={"upvotes": 450, "comments": 82, "simulated": True},
+                        )
+                    ]
+                all_signals.extend(social_signals)
+                self._source_health["social"]["status"] = "healthy"
+                self._source_health["social"]["last_success"] = datetime.utcnow().isoformat()
+            except Exception:
+                self._source_health["social"]["status"] = "unhealthy"
+                if config.ALLOW_SIMULATED_SOURCES:
+                    all_signals.extend(self._simulated_social_signals())
+
+        # Fetch from Blogs (labs + ecosystem)
+        if source in ("all", "blog"):
+            try:
+                blog_signals = []
+                if config.ENABLE_BLOG_REAL:
+                    blog_signals = await self.blog_crawler.fetch_blog_signals(
+                        max_results=config.BLOG_MAX_RESULTS
+                    )
+                all_signals.extend(blog_signals)
+                self._source_health["blog"]["status"] = "healthy"
+                self._source_health["blog"]["last_success"] = datetime.utcnow().isoformat()
+            except Exception:
+                self._source_health["blog"]["status"] = "unhealthy"
+
+        # Embed all signals
+        if all_signals:
+            texts = [
+                f"{s.title}. {s.raw_text}" for s in all_signals
+            ]
+            embeddings = self.embedding_engine.embed_batch(texts)
+
+            for signal, embedding in zip(all_signals, embeddings):
+                signal.embedding = embedding
+
+                # Compute novelty score
+                signal.novelty_score = self.vector_store.compute_novelty_score(
+                    embedding
+                )
+
+                # Store in vector store
+                self.vector_store.upsert_signal(
+                    signal_id=signal.id,
+                    embedding=embedding,
+                    payload={
+                        "id": signal.id,
+                        "source": signal.source.value,
+                        "source_id": signal.source_id,
+                        "title": signal.title,
+                        "raw_text": signal.raw_text[:500],
+                        "authors": signal.authors[:5],
+                        "categories": signal.categories,
+                        "url": signal.url,
+                        "novelty_score": signal.novelty_score,
+                        "timestamp": signal.timestamp.isoformat(),
+                        "metadata": signal.metadata,
+                    },
+                )
+
+                # Publish event for each new signal
+                await self.publish(
+                    "ingestion.new_signal",
+                    {
+                        "signal_id": signal.id,
+                        "source": signal.source.value,
+                        "title": signal.title,
+                        "novelty_score": signal.novelty_score,
+                    },
+                )
+
+        self._ingestion_count += len(all_signals)
+        self._last_ingestion = datetime.utcnow()
+        self._status = "running"
+
+        logger.info(
+            f"Ingestion complete: {len(all_signals)} signals "
+            f"(total: {self._ingestion_count})"
+        )
+
+        return all_signals
+
+    def get_health(self) -> dict:
+        health = super().get_health()
+        health.update({
+            "source_health": self._source_health,
+            "total_ingested": self._ingestion_count,
+            "last_ingestion": (
+                self._last_ingestion.isoformat() if self._last_ingestion else None
+            ),
+        })
+        return health
+
+    def _simulated_patent_signals(self) -> list[ResearchSignal]:
+        return [
+            ResearchSignal(
+                source=SignalSource.PATENT,
+                source_id="US-2026-0012345",
+                title="Distributed Multi-Agent Reasoning via Sparse Attention Meshes",
+                authors=["VectorMind R&D"],
+                raw_text="A method and system for optimizing multi-agent reasoning in decentralized networks...",
+                url="https://patents.google.com/patent/US20260012345A1",
+                metadata={
+                    "patent_number": "US20260012345",
+                    "assignee": "Samsung R&D",
+                    "simulated": True,
+                },
+            )
+        ]
+
+    def _simulated_startup_signals(self) -> list[ResearchSignal]:
+        return [
+            ResearchSignal(
+                source=SignalSource.STARTUP,
+                source_id="YC-W26-VECT",
+                title="Seed Funding: NeuroForge AI (YC W26)",
+                authors=["YC"],
+                raw_text="NeuroForge AI raises $5M to commercialize sparse attention architectures.",
+                url="https://ycombinator.com/companies/neuroforge",
+                metadata={"funding_round": "Seed", "amount": "$5M", "simulated": True},
+            )
+        ]
+
+    def _simulated_social_signals(self) -> list[ResearchSignal]:
+        return [
+            ResearchSignal(
+                source=SignalSource.SOCIAL,
+                source_id="HN-4123456",
+                title="Show HN: VectorMind - Open Source Research Intelligence",
+                authors=["hn_user"],
+                raw_text="The first agentic platform for autonomous research...",
+                url="https://news.ycombinator.com/item?id=4123456",
+                metadata={"upvotes": 450, "comments": 82, "simulated": True},
+            )
+        ]

agents/memory_agent.py

@@ -0,0 +1,144 @@
+"""Memory Agent — Long-horizon context and personalization.
+
+Stores user interaction history, manages feedback signals,
+and provides personalized scoring adjustments.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from datetime import datetime
+from typing import Optional
+
+from agents.base_agent import BaseAgent
+from ingestion.schema import AgentEvent, UserFeedback
+
+logger = logging.getLogger("vectorminds.memory_agent")
+
+
+class MemoryAgent(BaseAgent):
+    """Agent that maintains persistent context and user preferences."""
+
+    def __init__(self):
+        super().__init__("MemoryAgent")
+
+        # Working memory (replaces Redis)
+        self.working_memory: dict[str, dict] = {}
+
+        # Episodic memory — interaction history
+        self.interaction_history: list[dict] = []
+
+        # User preferences (learned from feedback)
+        self.user_preferences: dict[str, float] = defaultdict(float)
+
+        # Feedback store
+        self.feedback_log: list[UserFeedback] = []
+
+        # Blueprint cache
+        self.blueprint_cache: dict[str, dict] = {}
+
+        # Pipeline portfolio
+        self.pipeline_portfolio: list[dict] = []
+
+    def setup(self):
+        self.subscribe("reasoning.scored")
+        self.subscribe("delivery.feedback")
+
+    async def process_event(self, event: AgentEvent):
+        """Process scored signals and feedback events."""
+        if event.topic == "reasoning.scored":
+            # Store in episodic memory
+            self.interaction_history.append({
+                "type": "signal_scored",
+                "timestamp": datetime.utcnow().isoformat(),
+                "data": event.payload,
+            })
+            # Update working memory
+            signal_id = event.payload.get("signal_id", "")
+            if signal_id:
+                self.working_memory[signal_id] = event.payload
+
+        elif event.topic == "delivery.feedback":
+            await self._process_feedback(event.payload)
+
+    async def _process_feedback(self, payload: dict):
+        """Process user feedback to update preferences."""
+        feedback = UserFeedback(
+            target_id=payload.get("target_id", ""),
+            target_type=payload.get("target_type", "trend"),
+            action=payload.get("action", "upvote"),
+        )
+        self.feedback_log.append(feedback)
+
+        # Update user preferences based on feedback
+        categories = payload.get("categories", [])
+        weight = 1.0 if feedback.action == "upvote" else -0.5
+
+        for category in categories:
+            self.user_preferences[category] += weight
+
+        logger.info(
+            f"Feedback recorded: {feedback.action} on "
+            f"{feedback.target_type}/{feedback.target_id}"
+        )
+
+    def get_preference_weight(self, categories: list[str]) -> float:
+        """Get a personalization weight based on user preferences.
+
+        Args:
+            categories: List of categories for a signal/trend
+
+        Returns:
+            Weight multiplier (>1 = preferred, <1 = less preferred)
+        """
+        if not self.user_preferences:
+            return 1.0
+
+        scores = [self.user_preferences.get(c, 0) for c in categories]
+        if not scores:
+            return 1.0
+
+        avg_pref = sum(scores) / len(scores)
+        # Normalize to a multiplier around 1.0
+        return max(0.5, min(1.5, 1.0 + avg_pref * 0.1))
+
+    def store_blueprint(self, blueprint_id: str, data: dict):
+        """Cache a generated blueprint."""
+        self.blueprint_cache[blueprint_id] = {
+            **data,
+            "stored_at": datetime.utcnow().isoformat(),
+        }
+
+    def get_blueprint(self, blueprint_id: str) -> Optional[dict]:
+        """Retrieve a cached blueprint."""
+        return self.blueprint_cache.get(blueprint_id)
+
+    def store_pipeline(self, pipeline_data: dict):
+        """Add a pipeline to the portfolio."""
+        self.pipeline_portfolio.append({
+            **pipeline_data,
+            "stored_at": datetime.utcnow().isoformat(),
+        })
+
+    def get_stats(self) -> dict:
+        """Get memory agent statistics."""
+        return {
+            "working_memory_size": len(self.working_memory),
+            "interaction_count": len(self.interaction_history),
+            "feedback_count": len(self.feedback_log),
+            "blueprints_cached": len(self.blueprint_cache),
+            "pipelines_stored": len(self.pipeline_portfolio),
+            "preference_categories": len(self.user_preferences),
+            "upvotes": sum(
+                1 for f in self.feedback_log if f.action == "upvote"
+            ),
+            "downvotes": sum(
+                1 for f in self.feedback_log if f.action == "downvote"
+            ),
+        }
+
+    def get_health(self) -> dict:
+        health = super().get_health()
+        health.update(self.get_stats())
+        return health

agents/message_bus.py

@@ -0,0 +1,145 @@
+"""Async Message Bus — Inter-agent communication layer.
+
+Replaces Apache Kafka with asyncio queues for the hackathon MVP.
+Typed event system with pub/sub pattern.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import defaultdict
+from datetime import datetime
+from typing import Callable, Optional
+
+import config
+from ingestion.schema import AgentEvent
+
+logger = logging.getLogger("vectorminds.messagebus")
+
+try:
+    from aiokafka import AIOKafkaProducer
+except Exception:  # pragma: no cover - optional dependency
+    AIOKafkaProducer = None
+
+
+class MessageBus:
+    """In-memory async message bus for agent communication."""
+
+    _instance: Optional["MessageBus"] = None
+
+    @classmethod
+    def get_instance(cls) -> "MessageBus":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        self._subscribers: dict[str, list[asyncio.Queue]] = defaultdict(list)
+        self._handlers: dict[str, list[Callable]] = defaultdict(list)
+        self._event_log: list[AgentEvent] = []
+        self._running = False
+        self._producer = None
+        self._mirror_enabled = (
+            config.MESSAGE_BUS_BACKEND == "kafka_mirror" and AIOKafkaProducer is not None
+        )
+
+    async def start(self):
+        """Start optional Kafka mirror producer."""
+        if self._mirror_enabled and self._producer is None:
+            self._producer = AIOKafkaProducer(
+                bootstrap_servers=config.KAFKA_BOOTSTRAP_SERVERS,
+            )
+            await self._producer.start()
+            logger.info("Message bus Kafka mirror producer started")
+
+    async def stop(self):
+        """Stop optional Kafka producer."""
+        if self._producer is not None:
+            await self._producer.stop()
+            self._producer = None
+
+    def subscribe(self, topic: str) -> asyncio.Queue:
+        """Subscribe to a topic and get a queue for receiving events.
+
+        Args:
+            topic: Event topic (e.g. 'ingestion.new_signal')
+
+        Returns:
+            asyncio.Queue that will receive events for this topic
+        """
+        queue = asyncio.Queue()
+        self._subscribers[topic].append(queue)
+        logger.debug(f"New subscriber for topic '{topic}'")
+        return queue
+
+    def register_handler(self, topic: str, handler: Callable):
+        """Register a handler function for a topic.
+
+        Args:
+            topic: Event topic
+            handler: Async callable that processes AgentEvent
+        """
+        self._handlers[topic].append(handler)
+        logger.debug(f"Handler registered for topic '{topic}'")
+
+    async def publish(self, event: AgentEvent):
+        """Publish an event to all subscribers of its topic.
+
+        Args:
+            event: The event to publish
+        """
+        self._event_log.append(event)
+
+        # Send to queue subscribers
+        for queue in self._subscribers.get(event.topic, []):
+            await queue.put(event)
+
+        # Call registered handlers
+        for handler in self._handlers.get(event.topic, []):
+            try:
+                await handler(event)
+            except Exception as e:
+                logger.error(f"Handler error for topic '{event.topic}': {e}")
+
+        # Mirror events to Kafka if enabled.
+        if self._producer is not None:
+            kafka_topic = (
+                f"{config.KAFKA_TOPIC_PREFIX}."
+                f"{event.topic.replace('.', '_')}"
+            )
+            try:
+                await self._producer.send_and_wait(
+                    kafka_topic,
+                    event.model_dump_json().encode("utf-8"),
+                )
+            except Exception as e:
+                logger.error(f"Kafka mirror publish error for '{kafka_topic}': {e}")
+
+        logger.debug(
+            f"Published event: topic='{event.topic}', "
+            f"source='{event.source_agent}'"
+        )
+
+    async def publish_simple(self, topic: str, source: str, payload: dict):
+        """Convenience method to publish an event with minimal boilerplate."""
+        event = AgentEvent(
+            topic=topic,
+            source_agent=source,
+            timestamp=datetime.utcnow(),
+            payload=payload,
+        )
+        await self.publish(event)
+
+    def get_recent_events(self, topic: str = None, limit: int = 50) -> list[dict]:
+        """Get recent events, optionally filtered by topic."""
+        events = self._event_log
+        if topic:
+            events = [e for e in events if e.topic == topic]
+        return [e.model_dump() for e in events[-limit:]]
+
+    def clear(self):
+        """Clear all subscriptions and event log."""
+        self._subscribers.clear()
+        self._handlers.clear()
+        self._event_log.clear()

agents/reasoning_agent.py

@@ -0,0 +1,390 @@
+"""Reasoning Agent — Core intelligence agent of VectorMinds.
+
+Performs cross-source correlation, novelty scoring, impact prediction,
+and chain-of-thought summarization using LLM.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime
+from typing import Optional
+
+import httpx
+
+from agents.base_agent import BaseAgent
+from embeddings.vector_store import VectorStore
+from ingestion.schema import AgentEvent, TrendEntry
+import config
+
+logger = logging.getLogger("vectorminds.reasoning_agent")
+
+
+# ─── Mock LLM Responses (for demo when no API key) ───────────
+MOCK_BRIEFS = [
+    {
+        "technique": "State Space Models (Mamba)",
+        "brief": "A new class of sequence models that achieve transformer-quality results with linear-time complexity. Mamba introduces selective state spaces that can dynamically filter information, enabling 5x faster inference and the ability to process sequences of unlimited length. Key insight: by making the state space parameters input-dependent, the model can selectively remember or forget information.",
+        "impact": "Could fundamentally reshape the efficiency of language models, making GPT-4-class models runnable on consumer hardware.",
+        "competes_with": ["Transformers", "RWKV", "Hyena"],
+    },
+    {
+        "technique": "Mixture of Experts (MoE) Scaling",
+        "brief": "Sparse MoE architectures activate only a fraction of parameters per token, enabling models with trillions of parameters to run at the cost of much smaller dense models. Recent innovations in expert routing and load balancing have made MoE practical for production deployment.",
+        "impact": "Enables 10x parameter scaling without proportional compute increase. Key enabler for next-generation foundation models.",
+        "competes_with": ["Dense Transformers", "Distillation", "Pruning"],
+    },
+    {
+        "technique": "Retrieval-Augmented Generation (RAG) 2.0",
+        "brief": "Next-generation RAG systems move beyond simple vector similarity search. They incorporate graph-based retrieval, multi-hop reasoning chains, and learned retrieval strategies that adapt to the query complexity. Self-RAG and Corrective RAG add self-reflection loops that verify retrieved context relevance.",
+        "impact": "Dramatically reduces hallucination in production LLM applications. Enables enterprise AI deployment with verifiable sources.",
+        "competes_with": ["Fine-tuning", "In-context Learning", "Knowledge Graphs"],
+    },
+    {
+        "technique": "Diffusion Transformers (DiT)",
+        "brief": "Replacing the U-Net backbone in diffusion models with transformer architectures. DiT achieves state-of-the-art image generation quality while being more scalable and amenable to the same scaling laws that power language models.",
+        "impact": "Unifies the image and language generation paradigms under a single architecture family, enabling multimodal foundation models.",
+        "competes_with": ["U-Net Diffusion", "GANs", "Autoregressive Image Models"],
+    },
+    {
+        "technique": "Constitutional AI & RLHF Alternatives",
+        "brief": "New alignment approaches that reduce dependence on expensive human feedback. DPO (Direct Preference Optimization) eliminates the reward model entirely. Constitutional AI uses AI-generated critiques to self-improve. KTO (Kahneman-Tversky Optimization) requires only binary good/bad labels.",
+        "impact": "Democratizes model alignment — any team can align models without a large annotation workforce.",
+        "competes_with": ["RLHF", "PPO", "Manual Prompt Engineering"],
+    },
+    {
+        "technique": "Multi-Agent LLM Systems",
+        "brief": "Frameworks where multiple LLM instances collaborate as specialized agents to solve complex tasks. Each agent has defined roles, tools, and memory. Key innovations: hierarchical planning agents, critic/validator agents, and shared workspace protocols.",
+        "impact": "Moves AI from single-turn Q&A to autonomous multi-step problem solving. Foundation for AI software engineers and research assistants.",
+        "competes_with": ["Single-agent CoT", "Function Calling", "Manual Workflows"],
+    },
+    {
+        "technique": "LoRA & Parameter-Efficient Fine-Tuning",
+        "brief": "Low-Rank Adaptation enables fine-tuning large models by training only small adapter matrices. QLoRA extends this with 4-bit quantization, enabling fine-tuning of 65B models on a single GPU. DoRA and LoRA+ improve convergence and final quality.",
+        "impact": "Every organization can now customize foundation models for their domain at minimal cost. Enables private, specialized AI.",
+        "competes_with": ["Full Fine-tuning", "Prompt Tuning", "In-context Learning"],
+    },
+    {
+        "technique": "Vision-Language Models (VLMs)",
+        "brief": "Models that natively understand both images and text, enabling visual question answering, image-grounded dialogue, and document understanding. LLaVA, Qwen-VL, and GPT-4V demonstrate that vision encoders can be efficiently fused with language models.",
+        "impact": "Unlocks multimodal AI applications: automated document processing, visual inspection, accessibility tools, and creative design assistants.",
+        "competes_with": ["OCR + LLM Pipelines", "CLIP", "Specialized Vision Models"],
+    },
+    {
+        "technique": "Structured Output Generation",
+        "brief": "Constrained decoding techniques that guarantee LLM outputs conform to a specified schema (JSON, SQL, code). Grammar-guided generation and token masking ensure 100% format compliance without post-processing.",
+        "impact": "Makes LLMs reliable for production software integration. Eliminates the 'parsing problem' that plagues LLM-powered applications.",
+        "competes_with": ["Regex Post-processing", "Retry Loops", "Fine-tuning for Format"],
+    },
+    {
+        "technique": "Speculative Decoding & Inference Optimization",
+        "brief": "Techniques that accelerate LLM inference without quality loss. Speculative decoding uses a small draft model to propose tokens that a large model verifies in parallel. Combined with KV-cache optimization, flash attention, and quantization, these achieve 3-5x speedup.",
+        "impact": "Makes large model deployment economically viable. Reduces per-token cost to enable real-time conversational AI at scale.",
+        "competes_with": ["Model Distillation", "Pruning", "Smaller Models"],
+    },
+]
+
+
+class ReasoningAgent(BaseAgent):
+    """Agent that performs cross-source analysis, scoring, and summarization."""
+
+    def __init__(self):
+        super().__init__("ReasoningAgent")
+        self.vector_store = VectorStore.get_instance()
+        self.trends: dict[str, TrendEntry] = {}
+        self._mock_brief_idx = 0
+
+    def setup(self):
+        self.subscribe("ingestion.new_signal")
+
+    async def process_event(self, event: AgentEvent):
+        """Process new research signals — score and cluster them."""
+        if event.topic == "ingestion.new_signal":
+            signal_data = event.payload
+            logger.debug(
+                f"Processing signal: {signal_data.get('title', 'unknown')}"
+            )
+            # Impact scoring happens in batch via analyze_trends()
+
+    async def analyze_trends(self) -> list[TrendEntry]:
+        """Analyze all stored signals and produce a ranked trend leaderboard.
+
+        This is the core intelligence function that:
+        1. Clusters related signals
+        2. Scores each cluster for emergence and impact
+        3. Generates technical briefs
+        4. Returns ranked trends
+
+        Returns:
+            Sorted list of TrendEntry objects
+        """
+        self._status = "analyzing"
+        logger.info("Running trend analysis...")
+
+        payloads = self.vector_store.get_all_payloads(limit=500)
+        if not payloads:
+            self._status = "running"
+            return []
+
+        # Group signals by technique/topic (simplified clustering via categories)
+        technique_clusters: dict[str, list[dict]] = {}
+        for p in payloads:
+            # Extract primary technique from categories and title
+            categories = p.get("categories", [])
+            title = p.get("title", "")
+
+            # Use first category as cluster key, or title keywords
+            key = self._extract_technique_key(title, categories)
+            if key not in technique_clusters:
+                technique_clusters[key] = []
+            technique_clusters[key].append(p)
+
+        # Score each cluster
+        trends = []
+        for idx, (technique, signals) in enumerate(technique_clusters.items()):
+            novelty_scores = [s.get("novelty_score", 0) for s in signals]
+            avg_novelty = sum(novelty_scores) / len(novelty_scores) if novelty_scores else 0
+
+            # Compute emergence score
+            github_stars = sum(
+                s.get("metadata", {}).get("stars", 0)
+                for s in signals
+                if s.get("source") == "github"
+            )
+            paper_count = sum(1 for s in signals if s.get("source") == "arxiv")
+
+            emergence_score = self._compute_emergence_score(
+                avg_novelty, len(signals), github_stars, paper_count
+            )
+
+            # Impact prediction (simplified heuristic for MVP)
+            impact_score = self._predict_impact(
+                avg_novelty, len(signals), github_stars
+            )
+
+            # Get or generate technical brief
+            brief_data = self._get_mock_brief(idx)
+
+            trend = TrendEntry(
+                rank=0,  # Will be set after sorting
+                technique_name=brief_data["technique"] if config.USE_MOCK_LLM else technique,
+                description=brief_data["brief"],
+                emergence_score=round(emergence_score, 3),
+                novelty_score=round(avg_novelty, 3),
+                impact_score=round(impact_score, 3),
+                mainstream_eta_months=self._estimate_eta(impact_score),
+                confidence=round(min(0.95, 0.5 + impact_score * 0.4), 2),
+                source_signals={
+                    "arxiv_papers": paper_count,
+                    "github_repos": len(signals) - paper_count,
+                    "total_github_stars": github_stars,
+                },
+                competitive_landscape=brief_data.get("competes_with", []),
+                risk_factors=self._assess_risks(signals),
+                related_techniques=brief_data.get("competes_with", [])[:3],
+                paper_count=paper_count,
+                github_stars=github_stars,
+                signal_ids=[s.get("id", "") for s in signals],
+            )
+            trends.append(trend)
+
+        # Sort by emergence score and assign ranks
+        trends.sort(key=lambda t: t.emergence_score, reverse=True)
+        for i, trend in enumerate(trends):
+            trend.rank = i + 1
+
+        # Store trends
+        self.trends = {t.id: t for t in trends}
+
+        self._status = "running"
+        logger.info(f"Trend analysis complete: {len(trends)} techniques ranked")
+        return trends[:20]  # Return top 20
+
+    async def generate_technical_brief(
+        self, technique: str, context: str = ""
+    ) -> str:
+        """Generate a detailed technical brief using LLM or mock.
+
+        Args:
+            technique: Technique name
+            context: Additional context (paper abstracts, etc.)
+
+        Returns:
+            Formatted technical brief text
+        """
+        if config.USE_MOCK_LLM or not config.LLM_API_KEY:
+            return self._generate_mock_brief(technique)
+
+        try:
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                prompt = (
+                    "You are a senior AI research analyst at VectorMinds. "
+                    "Generate a structured technical brief for an emerging AI technique. "
+                    "Include: core technique description, key insight, why it matters, "
+                    "what it enables, what it competes with, and 12-month impact prediction.\n\n"
+                    f"Technique: {technique}\n\n"
+                    f"Context from recent papers:\n{context[:2000]}"
+                )
+                resp = await client.post(
+                    f"{config.GEMINI_BASE_URL}/models/{config.LLM_MODEL}:generateContent",
+                    params={"key": config.LLM_API_KEY},
+                    headers={"Content-Type": "application/json"},
+                    json={
+                        "contents": [{"parts": [{"text": prompt}]}],
+                        "generationConfig": {
+                            "temperature": 0.7,
+                            "maxOutputTokens": 4096,
+                        },
+                    },
+                )
+                resp.raise_for_status()
+                data = resp.json()
+                candidates = data.get("candidates", [])
+                if not candidates:
+                    raise ValueError("No Gemini candidates returned")
+                parts = candidates[0].get("content", {}).get("parts", [])
+                text = "".join(p.get("text", "") for p in parts if isinstance(p, dict)).strip()
+                if not text:
+                    raise ValueError("Empty Gemini response text")
+                return text
+        except Exception as e:
+            logger.error(f"LLM brief generation failed: {e}")
+            return self._generate_mock_brief(technique)
+
+    def _extract_technique_key(self, title: str, categories: list) -> str:
+        """Extract a technique cluster key from title and categories.
+
+        Priority:
+        1. Match a curated keyword list against the title.
+        2. Match against descriptive category tokens (skipping arXiv class codes
+           like ``cs.LG`` and source labels like ``arxiv``/``blog``).
+        3. Fallback to ``General AI`` so we never surface an internal source label
+           as a "technique".
+        """
+        title_lower = (title or "").lower()
+
+        keywords = [
+            ("transformer", "Transformer Architectures"),
+            ("attention", "Attention Mechanisms"),
+            ("diffusion", "Diffusion Models"),
+            ("rlhf", "RLHF Alignment"),
+            ("reinforcement", "Reinforcement Learning"),
+            ("graph neural", "Graph Neural Networks"),
+            ("federated", "Federated Learning"),
+            ("quantization", "Quantization"),
+            ("pruning", "Model Pruning"),
+            ("distillation", "Knowledge Distillation"),
+            ("mixture of experts", "Mixture of Experts"),
+            ("moe", "Mixture of Experts"),
+            ("retrieval-augmented", "Retrieval-Augmented Generation"),
+            ("rag", "Retrieval-Augmented Generation"),
+            ("lora", "LoRA / Adapter Tuning"),
+            ("fine-tun", "Parameter-Efficient Fine-Tuning"),
+            ("multimodal", "Multimodal Models"),
+            ("vision-language", "Vision-Language Models"),
+            ("vision language", "Vision-Language Models"),
+            ("vision", "Vision Models"),
+            ("agent", "AI Agents"),
+            ("state space", "State Space Models"),
+            ("mamba", "State Space Models"),
+            ("language model", "Large Language Models"),
+            ("llm", "Large Language Models"),
+            ("embedding", "Embedding Models"),
+            ("contrastive", "Contrastive Learning"),
+            ("self-supervised", "Self-Supervised Learning"),
+            ("gan", "Generative Adversarial Networks"),
+            ("speech", "Speech Models"),
+            ("audio", "Audio Models"),
+        ]
+
+        for kw, label in keywords:
+            if kw in title_lower:
+                return label
+
+        # Categories: ignore arXiv class codes (e.g. cs.LG) and source labels.
+        IGNORE_CATEGORY_LABELS = {
+            "arxiv", "github", "patents", "patent", "startup", "startups",
+            "social", "blog", "blogs", "hacker-news", "rss",
+        }
+        if categories:
+            for cat in categories:
+                if not isinstance(cat, str):
+                    continue
+                low = cat.lower().strip()
+                if not low:
+                    continue
+                if low in IGNORE_CATEGORY_LABELS:
+                    continue
+                # Skip arXiv subject codes such as cs.LG / stat.ML.
+                if "." in low and len(low) <= 8 and low.split(".")[0].isalpha():
+                    continue
+                return cat.replace("-", " ").title()
+
+        return "General AI"
+
+    def _compute_emergence_score(
+        self,
+        avg_novelty: float,
+        signal_count: int,
+        github_stars: int,
+        paper_count: int,
+    ) -> float:
+        """Compute emergence score (composite metric)."""
+        # Weighted combination of multiple signals
+        novelty_component = avg_novelty * 0.35
+        volume_component = min(1.0, signal_count / 20) * 0.25
+        github_component = min(1.0, github_stars / 5000) * 0.25
+        academic_component = min(1.0, paper_count / 10) * 0.15
+
+        return novelty_component + volume_component + github_component + academic_component
+
+    def _predict_impact(
+        self, avg_novelty: float, signal_count: int, github_stars: int
+    ) -> float:
+        """Simplified impact prediction (MVP heuristic)."""
+        # In production, this would be an XGBoost ensemble
+        base = avg_novelty * 0.4
+        volume_signal = min(1.0, signal_count / 15) * 0.3
+        community_signal = min(1.0, github_stars / 3000) * 0.3
+        return max(0.0, min(1.0, base + volume_signal + community_signal))
+
+    def _estimate_eta(self, impact_score: float) -> int:
+        """Estimate deterministic mainstream horizon buckets (6/12/24 months)."""
+        if impact_score >= 0.75:
+            return 6
+        if impact_score >= 0.5:
+            return 12
+        return 24
+
+    def _assess_risks(self, signals: list[dict]) -> list[str]:
+        """Generate risk factors for a technique cluster."""
+        risks = []
+        paper_count = sum(1 for s in signals if s.get("source") == "arxiv")
+
+        if paper_count < 3:
+            risks.append("Limited academic validation (few papers)")
+        if not any(s.get("source") == "github" for s in signals):
+            risks.append("No open-source implementations detected")
+
+        # Default risks
+        risks.extend([
+            "Compute requirements may limit accessibility",
+            "Dataset licensing considerations for commercial use",
+        ])
+        return risks[:4]
+
+    def _get_mock_brief(self, idx: int) -> dict:
+        """Get a pre-written mock brief for demo."""
+        return MOCK_BRIEFS[idx % len(MOCK_BRIEFS)]
+
+    def _generate_mock_brief(self, technique: str) -> str:
+        """Generate a mock technical brief."""
+        self._mock_brief_idx = (self._mock_brief_idx + 1) % len(MOCK_BRIEFS)
+        brief = MOCK_BRIEFS[self._mock_brief_idx]
+        return (
+            f"## Technical Brief: {technique}\n\n"
+            f"**Core Technique:** {brief['brief']}\n\n"
+            f"**Impact:** {brief['impact']}\n\n"
+            f"**Competes With:** {', '.join(brief['competes_with'])}"
+        )

agents/retraining_agent.py

@@ -0,0 +1,149 @@
+"""Retraining Agent — Autonomous Model Drift & Promotion Engine.
+
+Follows Section 4.3: Monitors for model drift and orchestrates
+automated retraining/promotion cycles.
+"""
+
+import logging
+import asyncio
+from datetime import datetime
+import numpy as np
+
+from agents.base_agent import BaseAgent
+from ingestion.schema import AgentEvent
+import config
+
+logger = logging.getLogger("vectormind.retraining")
+
+class RetrainingAgent(BaseAgent):
+    """Agent that handles autonomous model maintenance."""
+
+    def __init__(self):
+        super().__init__("RetrainingAgent")
+        self.last_retraining = None
+        self.drift_history = []
+        self._status = "monitoring"
+        self.model_version = "v1.1.0"
+        self._baseline_metrics = {
+            "accuracy": 0.78,
+            "f1": 0.75,
+            "latency_ms": 180.0,
+        }
+        self._last_candidate_metrics = None
+
+    def setup(self):
+        # Subscribe to new signals to monitor drift
+        self.subscribe("ingestion.new_signal")
+
+    async def process_event(self, event: AgentEvent):
+        """Monitor incoming signals for novelty distribution drift."""
+        if event.topic == "ingestion.new_signal":
+            novelty_score = event.payload.get("novelty_score", 0)
+            self.drift_history.append(novelty_score)
+            
+            # Check for drift every 100 signals
+            if len(self.drift_history) >= 100:
+                await self.check_drift()
+
+    async def check_drift(self):
+        """Analyze novelty distribution to detect model staleness."""
+        avg_novelty = np.mean(self.drift_history)
+        logger.info(f"Checking model drift. Avg Novelty: {avg_novelty:.4f}")
+        
+        # If novelty is too low, it means our vector store is too crowded 
+        # with similar content, and the model might need recalibration.
+        if avg_novelty < config.RETRAIN_DRIFT_THRESHOLD:
+            logger.warning("Significant model drift detected! Novelty threshold breached.")
+            await self.trigger_retraining()
+            
+        self.drift_history = []
+
+    def _evaluate_quality_gates(self, candidate_metrics: dict) -> tuple[bool, list[str]]:
+        """Evaluate if candidate model satisfies production promotion rules."""
+        reasons = []
+        accuracy = float(candidate_metrics.get("accuracy", 0.0))
+        f1 = float(candidate_metrics.get("f1", 0.0))
+        latency = float(candidate_metrics.get("latency_ms", 10_000.0))
+
+        if accuracy < config.RETRAIN_MIN_ACCURACY:
+            reasons.append(f"accuracy below threshold ({accuracy:.3f} < {config.RETRAIN_MIN_ACCURACY:.3f})")
+        if f1 < config.RETRAIN_MIN_F1:
+            reasons.append(f"f1 below threshold ({f1:.3f} < {config.RETRAIN_MIN_F1:.3f})")
+        if latency > config.RETRAIN_MAX_LATENCY_MS:
+            reasons.append(f"latency above threshold ({latency:.1f} > {config.RETRAIN_MAX_LATENCY_MS:.1f})")
+
+        baseline_accuracy = float(self._baseline_metrics.get("accuracy", 0.0))
+        if (accuracy - baseline_accuracy) < config.RETRAIN_MIN_IMPROVEMENT:
+            reasons.append(
+                f"accuracy improvement too small ({accuracy - baseline_accuracy:.3f} < {config.RETRAIN_MIN_IMPROVEMENT:.3f})"
+            )
+        return len(reasons) == 0, reasons
+
+    def _next_model_version(self, promoted: bool) -> str:
+        """Generate next semantic-like version according to promotion result."""
+        parts = self.model_version.lstrip("v").split(".")
+        major, minor, patch = [int(p) for p in (parts + ["0", "0", "0"])[:3]]
+        if promoted:
+            minor += 1
+            patch = 0
+        else:
+            patch += 1
+        return f"v{major}.{minor}.{patch}"
+
+    async def trigger_retraining(self, candidate_metrics: dict | None = None):
+        """Orchestrate the retraining and promotion cycle."""
+        self._status = "retraining"
+        logger.info("Initiating autonomous retraining cycle...")
+        
+        # 1. Snapshot vector store
+        # 2. Re-calculate embeddings with updated context
+        # 3. Validate new model performance
+        
+        await asyncio.sleep(2)
+
+        if candidate_metrics is None:
+            # Non-mock deterministic default candidate using recent drift context.
+            avg_novelty = float(np.mean(self.drift_history)) if self.drift_history else 0.25
+            candidate_metrics = {
+                "accuracy": round(max(0.7, 0.84 - (0.4 - min(avg_novelty, 0.4))), 3),
+                "f1": 0.78,
+                "latency_ms": 145.0,
+            }
+        self._last_candidate_metrics = candidate_metrics
+        passed, reasons = self._evaluate_quality_gates(candidate_metrics)
+
+        self.last_retraining = datetime.utcnow()
+        self._status = "monitoring"
+        next_version = self._next_model_version(promoted=passed)
+
+        if passed:
+            self.model_version = next_version
+            self._baseline_metrics = candidate_metrics
+            logger.info("Retraining complete. New model promoted to production.")
+            await self.publish("model.promoted", {
+                "timestamp": self.last_retraining.isoformat(),
+                "new_version": self.model_version,
+                "metrics": candidate_metrics,
+                "promotion_policy": "quality_gates_passed",
+            })
+        else:
+            self.model_version = next_version
+            logger.warning("Retraining candidate rejected by quality gates: %s", "; ".join(reasons))
+            await self.publish("model.retraining_failed", {
+                "timestamp": self.last_retraining.isoformat(),
+                "candidate_version": self.model_version,
+                "metrics": candidate_metrics,
+                "reasons": reasons,
+                "promotion_policy": "quality_gates_failed",
+            })
+
+    def get_health(self) -> dict:
+        health = super().get_health()
+        health.update({
+            "last_retraining": self.last_retraining.isoformat() if self.last_retraining else None,
+            "drift_status": "stable" if len(self.drift_history) < 50 else "analyzing",
+            "model_version": self.model_version,
+            "baseline_metrics": self._baseline_metrics,
+            "last_candidate_metrics": self._last_candidate_metrics,
+        })
+        return health

config.py

@@ -0,0 +1,132 @@
+"""VectorMind Configuration Module.
+
+Central configuration for all platform settings, loaded from environment
+variables with sensible defaults for hackathon demo mode.
+"""
+
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+# ─── Base Paths ───────────────────────────────────────────────
+BASE_DIR = Path(__file__).resolve().parent
+
+# When running on Hugging Face Spaces, /data is the only writable persistent
+# mount. Falling back to backend/data keeps local dev unchanged.
+_HF_DATA = Path("/data")
+DATA_DIR = (
+    _HF_DATA
+    if os.getenv("HF_DEPLOYMENT", "").lower() == "true" and _HF_DATA.exists()
+    else BASE_DIR / "data"
+)
+DATA_DIR.mkdir(parents=True, exist_ok=True)
+DB_PATH = DATA_DIR / "vectormind.db"
+
+# ─── API Keys (ALL FREE) ─────────────────────────────────────
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")
+# Backward-compatible fallback so older .env files still run.
+LLM_API_KEY = GEMINI_API_KEY or os.getenv("GROQ_API_KEY", "")
+TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "")
+TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "")
+HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN", "")
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")  # optional, for higher rate limits
+KAGGLE_USERNAME = os.getenv("KAGGLE_USERNAME", "")
+KAGGLE_KEY = os.getenv("KAGGLE_KEY", "")
+
+# ─── LLM Settings (Gemini) ───────────────────────────────────
+GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
+LLM_MODEL = os.getenv("LLM_MODEL", "gemini-2.5-flash-preview-05-20")
+USE_MOCK_LLM = os.getenv("USE_MOCK_LLM", "true").lower() == "true"
+
+# ─── Embedding Model (Free — runs locally) ───────────────────
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "BAAI/bge-small-en-v1.5")
+EMBEDDING_DIM = 384  # bge-small-en-v1.5 dimension
+
+# ─── Qdrant Settings (In-Memory — Free) ──────────────────────
+QDRANT_COLLECTION = "research_signals"
+QDRANT_HOST = os.getenv("QDRANT_HOST", "")  # empty = in-memory mode
+QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
+
+# ─── Distributed Infra (Docker/local) ────────────────────────
+DB_BACKEND = os.getenv("DB_BACKEND", "sqlite").lower()  # sqlite|postgres
+POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
+POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", "5432"))
+POSTGRES_DB = os.getenv("POSTGRES_DB", "vectormind")
+POSTGRES_USER = os.getenv("POSTGRES_USER", "vectormind")
+POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", "vectormind")
+POSTGRES_DSN = os.getenv(
+    "POSTGRES_DSN",
+    f"postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}",
+)
+
+REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
+STATE_STORE_BACKEND = os.getenv("STATE_STORE_BACKEND", "sqlite").lower()  # sqlite|redis
+
+MESSAGE_BUS_BACKEND = os.getenv("MESSAGE_BUS_BACKEND", "in_memory").lower()  # in_memory|kafka_mirror
+KAFKA_BOOTSTRAP_SERVERS = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092")
+KAFKA_TOPIC_PREFIX = os.getenv("KAFKA_TOPIC_PREFIX", "vectormind")
+
+# ─── Pipeline Runtime (production knobs) ─────────────────────
+PIPELINE_RUN_TIMEOUT_SECONDS = int(os.getenv("PIPELINE_RUN_TIMEOUT_SECONDS", "1800"))
+PIPELINE_MAX_CONCURRENT_RUNS = int(os.getenv("PIPELINE_MAX_CONCURRENT_RUNS", "2"))
+PIPELINE_MAX_RETRIES = int(os.getenv("PIPELINE_MAX_RETRIES", "1"))
+PIPELINE_RETRY_BACKOFF_SECONDS = int(os.getenv("PIPELINE_RETRY_BACKOFF_SECONDS", "5"))
+
+# ─── Novelty Scoring (Section 4.2) ───────────────────────────
+NOVELTY_K_NEIGHBORS = 50
+NOVELTY_MEAN_WEIGHT = 0.6
+NOVELTY_MIN_WEIGHT = 0.4
+NOVELTY_TEMPORAL_DISCOUNT = 0.7
+NOVELTY_TEMPORAL_WINDOW_HOURS = 72
+
+# ─── Impact Prediction ───────────────────────────────────────
+IMPACT_HIGH_THRESHOLD = 0.75
+IMPACT_MEDIUM_THRESHOLD = 0.50
+
+# ─── Retraining Promotion Gates ───────────────────────────────
+RETRAIN_DRIFT_THRESHOLD = float(os.getenv("RETRAIN_DRIFT_THRESHOLD", "0.30"))
+RETRAIN_MIN_ACCURACY = float(os.getenv("RETRAIN_MIN_ACCURACY", "0.78"))
+RETRAIN_MIN_F1 = float(os.getenv("RETRAIN_MIN_F1", "0.75"))
+RETRAIN_MAX_LATENCY_MS = float(os.getenv("RETRAIN_MAX_LATENCY_MS", "180"))
+RETRAIN_MIN_IMPROVEMENT = float(os.getenv("RETRAIN_MIN_IMPROVEMENT", "0.01"))
+
+# ─── Ingestion Settings ─────────────────────���────────────────
+ARXIV_CATEGORIES = ["cs.LG", "cs.AI", "cs.CL", "cs.CV", "cs.NE"]
+ARXIV_MAX_RESULTS = 50
+GITHUB_TRENDING_LANGUAGES = ["python", "jupyter-notebook"]
+GITHUB_MAX_RESULTS = 30
+INGESTION_INTERVAL_SECONDS = 3600  # 1 hour
+PATENTS_MAX_RESULTS = int(os.getenv("PATENTS_MAX_RESULTS", "20"))
+STARTUPS_MAX_RESULTS = int(os.getenv("STARTUPS_MAX_RESULTS", "20"))
+SOCIAL_MAX_RESULTS = int(os.getenv("SOCIAL_MAX_RESULTS", "30"))
+BLOG_MAX_RESULTS = int(os.getenv("BLOG_MAX_RESULTS", "20"))
+
+# External source controls
+ENABLE_PATENTS_REAL = os.getenv("ENABLE_PATENTS_REAL", "true").lower() == "true"
+ENABLE_STARTUPS_REAL = os.getenv("ENABLE_STARTUPS_REAL", "true").lower() == "true"
+ENABLE_SOCIAL_REAL = os.getenv("ENABLE_SOCIAL_REAL", "true").lower() == "true"
+ENABLE_BLOG_REAL = os.getenv("ENABLE_BLOG_REAL", "true").lower() == "true"
+ALLOW_SIMULATED_SOURCES = os.getenv("ALLOW_SIMULATED_SOURCES", "true").lower() == "true"
+
+# ─── Deduplication ────────────────────────────────────────────
+DEDUP_SIMILARITY_THRESHOLD = 0.95
+
+# ─── Server Settings ─────────────────────────────────────────
+API_HOST = os.getenv("API_HOST", "0.0.0.0")
+API_PORT = int(os.getenv("API_PORT", "8000"))
+API_ADMIN_KEY = os.getenv("API_ADMIN_KEY", "")
+# Allow any origin by default. Browsers enforce CORS, the Android client
+# isn't a browser, and we don't use cookies for auth — so a permissive
+# wildcard is fine and lets judges curl the API from anywhere.
+CORS_ORIGINS = os.getenv("CORS_ORIGINS", "*").split(",")
+
+# ─── Pipeline Generator ──────────────────────────────────────
+SUPPORTED_TASK_CATEGORIES = [
+    "text-classification",
+    "image-classification",
+    "text-generation",
+    "question-answering",
+    "summarization",
+]

data/pipeline_runs/147b5293-7c7b-4c13-85ff-b36a5208d85d/6c07740b-6b6f-4e74-9fec-7d72cee11a1c/pipeline.py

@@ -0,0 +1,5 @@
+import os
+import subprocess
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/147b5293-7c7b-4c13-85ff-b36a5208d85d/6c07740b-6b6f-4e74-9fec-7d72cee11a1c/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/147b5293-7c7b-4c13-85ff-b36a5208d85d/6c07740b-6b6f-4e74-9fec-7d72cee11a1c/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/1eb5fe5c-8c4b-4478-93fc-7b0502ebc54b/ceba6666-3bc5-42f3-b27a-60b8940793e5/pipeline.py

@@ -0,0 +1,5 @@
+import os
+import subprocess
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/1eb5fe5c-8c4b-4478-93fc-7b0502ebc54b/ceba6666-3bc5-42f3-b27a-60b8940793e5/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/1eb5fe5c-8c4b-4478-93fc-7b0502ebc54b/ceba6666-3bc5-42f3-b27a-60b8940793e5/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/494ff8ba-8b86-453a-8b40-750c5e34a634/d384b981-2582-466c-9524-8a101a5ec944/pipeline.py

@@ -0,0 +1,5 @@
+import os
+import subprocess
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/494ff8ba-8b86-453a-8b40-750c5e34a634/d384b981-2582-466c-9524-8a101a5ec944/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/494ff8ba-8b86-453a-8b40-750c5e34a634/d384b981-2582-466c-9524-8a101a5ec944/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/4a8279c0-16db-4713-bfca-b83f2b7a4e53/1d806c97-a4b8-4961-b1a3-bf3bf2e36996/pipeline.py

@@ -0,0 +1,5 @@
+import os
+import subprocess
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/4a8279c0-16db-4713-bfca-b83f2b7a4e53/1d806c97-a4b8-4961-b1a3-bf3bf2e36996/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/4a8279c0-16db-4713-bfca-b83f2b7a4e53/1d806c97-a4b8-4961-b1a3-bf3bf2e36996/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/83e14744-9a0a-4ece-829d-e9ad0dba3cbf/bda049c6-7392-4adf-853b-b59d66ade0fe/pipeline.py

@@ -0,0 +1,5 @@
+import os
+import subprocess
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/83e14744-9a0a-4ece-829d-e9ad0dba3cbf/bda049c6-7392-4adf-853b-b59d66ade0fe/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/83e14744-9a0a-4ece-829d-e9ad0dba3cbf/bda049c6-7392-4adf-853b-b59d66ade0fe/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/92ceb65d-565f-457c-a099-5f2fff00f093/39e0cfab-b5c4-4e03-ba91-9e3ce56a4482/pipeline.py

@@ -0,0 +1,4 @@
+import os
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/92ceb65d-565f-457c-a099-5f2fff00f093/39e0cfab-b5c4-4e03-ba91-9e3ce56a4482/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/92ceb65d-565f-457c-a099-5f2fff00f093/39e0cfab-b5c4-4e03-ba91-9e3ce56a4482/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

data/pipeline_runs/d30f2c12-4c47-40db-ba18-a7edd989515c/68664895-2b01-4fc4-9465-9413515b0d90/pipeline.py

@@ -0,0 +1,92 @@
+import os
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/d30f2c12-4c47-40db-ba18-a7edd989515c/68664895-2b01-4fc4-9465-9413515b0d90/artifacts')
+
+# VectorMind Autonomous Pipeline
+# Generated: 2026-05-08 06:09 UTC
+# Technique: Prod Runtime Smoke
+# Task: tabular-classification
+# Dataset: ag_news (huggingface-fallback)
+# Model: xgboost
+
+!pip install -q transformers datasets accelerate optuna onnx safetensors evaluate torch pillow
+
+
+# --- Dataset resolution ---
+DATASET_NAME = "ag_news"
+DATASET_SOURCE = "huggingface-fallback"
+MODEL_NAME = "xgboost"
+
+
+# --- Tabular baseline (executable sklearn pipeline) ---
+import numpy as np
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import HistGradientBoostingClassifier
+from sklearn.metrics import accuracy_score
+
+X, y = load_iris(return_X_y=True)
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
+clf = HistGradientBoostingClassifier(max_iter=50, random_state=42)
+clf.fit(X_train, y_train)
+pred = clf.predict(X_test)
+eval_metrics = {"eval_accuracy": float(accuracy_score(y_test, pred))}
+print("tabular metrics:", eval_metrics)
+
+
+# --- Bayesian Hyperparameter Optimization (Optuna) ---
+import optuna
+
+def objective(trial):
+    lr = trial.suggest_float("learning_rate", 1e-5, 5e-4, log=True)
+    batch = trial.suggest_categorical("batch_size", [4, 8, 16])
+    base = float(eval_metrics.get("eval_accuracy", eval_metrics.get("eval_loss", 0.0)))
+    if "loss" in str(eval_metrics):
+        return base - batch / 10000.0
+    return base - batch / 1000.0
+
+study = optuna.create_study(direction="maximize")
+study.optimize(objective, n_trials=5)
+print("Best hyperparameters:", study.best_params)
+
+
+# --- Multi-Format Export (SafeTensors; ONNX when compatible) ---
+from pathlib import Path
+import torch
+
+output_path = Path("./vectormind_export")
+output_path.mkdir(exist_ok=True)
+
+try:
+    model.save_pretrained(output_path, safe_serialization=True)
+    if "tokenizer" in dir():
+        tokenizer.save_pretrained(output_path)
+except Exception as ex:
+    print("save_pretrained skip:", ex)
+
+try:
+    dummy = {"input_ids": torch.ones(1, 8, dtype=torch.long), "attention_mask": torch.ones(1, 8, dtype=torch.long)}
+    if hasattr(model, "forward") and "input_ids" in dummy:
+        torch.onnx.export(
+            model,
+            (dummy["input_ids"], dummy["attention_mask"]),
+            output_path / "model.onnx",
+            input_names=["input_ids", "attention_mask"],
+            output_names=["logits"],
+            dynamic_axes={"input_ids": {0: "batch"}, "attention_mask": {0: "batch"}, "logits": {0: "batch"}},
+            opset_version=14,
+        )
+except Exception as ex:
+    print("ONNX export skipped (model may be vision/audio — use native torch.jit or HF optimum):", ex)
+
+try:
+    with open(output_path / "app.py", "w", encoding="utf-8") as f:
+        f.write("from fastapi import FastAPI\n")
+        f.write("app = FastAPI()\n")
+        f.write("@app.post('/predict')\n")
+        f.write("def predict(): return {'status': 'ok'}\n")
+except Exception:
+    pass
+
+print("Artifacts (partial):", output_path)
+
+print('Pipeline generation for tabular-classification completed.')

data/pipeline_runs/d30f2c12-4c47-40db-ba18-a7edd989515c/68664895-2b01-4fc4-9465-9413515b0d90/run.log

@@ -0,0 +1,4 @@
+  File "D:\zamzung\backend\data\pipeline_runs\d30f2c12-4c47-40db-ba18-a7edd989515c\68664895-2b01-4fc4-9465-9413515b0d90\pipeline.py", line 11
+    !pip install -q transformers datasets accelerate optuna onnx safetensors evaluate torch pillow
+    ^
+SyntaxError: invalid syntax

data/pipeline_runs/eb542fc3-721b-4cdd-ac4c-0c54f62ca512/d255fb9e-b352-47a5-b055-77f1b70709dd/pipeline.py

@@ -0,0 +1,4 @@
+import os
+os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'D:/zamzung/backend/data/pipeline_runs/eb542fc3-721b-4cdd-ac4c-0c54f62ca512/d255fb9e-b352-47a5-b055-77f1b70709dd/artifacts')
+
+print('pipeline-run-smoke')

data/pipeline_runs/eb542fc3-721b-4cdd-ac4c-0c54f62ca512/d255fb9e-b352-47a5-b055-77f1b70709dd/run.log

@@ -0,0 +1 @@
+pipeline-run-smoke

db/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Database Package

db/database.py

@@ -0,0 +1,527 @@
+"""Database — SQLite metadata storage layer.
+
+Replaces PostgreSQL for the hackathon MVP. Stores structured metadata,
+prediction records, user feedback, and agent state.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from typing import Optional
+
+import config
+
+logger = logging.getLogger("vectorminds.database")
+
+try:
+    import psycopg2
+except Exception:  # pragma: no cover - optional dependency
+    psycopg2 = None
+
+
+class Database:
+    """Metadata store for VectorMinds (SQLite or PostgreSQL)."""
+
+    _instance: Optional["Database"] = None
+
+    @classmethod
+    def get_instance(cls) -> "Database":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        self.db_path = str(config.DB_PATH)
+        self.backend = config.DB_BACKEND
+        self._conn: Optional[object] = None
+
+    def initialize(self):
+        """Create database and tables."""
+        if self.backend == "postgres":
+            if psycopg2 is None:
+                raise RuntimeError(
+                    "DB_BACKEND=postgres requires psycopg2-binary dependency."
+                )
+            self._conn = psycopg2.connect(config.POSTGRES_DSN)
+            self._conn.autocommit = False
+        else:
+            self._conn = sqlite3.connect(self.db_path, check_same_thread=False)
+            self._conn.row_factory = sqlite3.Row
+        self._create_tables()
+        logger.info(
+            "Database initialized using %s backend",
+            self.backend,
+        )
+
+    def _execute(self, query: str, params: tuple = ()):
+        cursor = self._conn.cursor()
+        cursor.execute(query, params)
+        return cursor
+
+    def _commit(self):
+        self._conn.commit()
+
+    def _placeholder(self) -> str:
+        return "%s" if self.backend == "postgres" else "?"
+
+    def _create_tables(self):
+        cursor = self._conn.cursor()
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS research_signals (
+                id TEXT PRIMARY KEY,
+                source TEXT NOT NULL,
+                source_id TEXT,
+                title TEXT NOT NULL,
+                raw_text TEXT,
+                authors TEXT,
+                categories TEXT,
+                url TEXT,
+                novelty_score REAL DEFAULT 0,
+                impact_score REAL DEFAULT 0,
+                metadata TEXT,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS trends (
+                id TEXT PRIMARY KEY,
+                rank INTEGER,
+                technique_name TEXT NOT NULL,
+                description TEXT,
+                emergence_score REAL DEFAULT 0,
+                novelty_score REAL DEFAULT 0,
+                impact_score REAL DEFAULT 0,
+                mainstream_eta_months INTEGER DEFAULT 12,
+                confidence REAL DEFAULT 0,
+                data TEXT,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS blueprints (
+                id TEXT PRIMARY KEY,
+                technique_name TEXT NOT NULL,
+                trend_id TEXT,
+                data TEXT,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS pipelines (
+                id TEXT PRIMARY KEY,
+                technique_name TEXT NOT NULL,
+                task_type TEXT,
+                status TEXT DEFAULT 'generated',
+                data TEXT,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS pipeline_runs (
+                run_id TEXT PRIMARY KEY,
+                pipeline_id TEXT NOT NULL,
+                status TEXT NOT NULL,
+                data TEXT,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS user_feedback (
+                id TEXT PRIMARY KEY,
+                target_id TEXT NOT NULL,
+                target_type TEXT NOT NULL,
+                action TEXT NOT NULL,
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS agent_state (
+                agent_name TEXT PRIMARY KEY,
+                state TEXT,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS telegram_subscribers (
+                chat_id BIGINT PRIMARY KEY,
+                username TEXT DEFAULT '',
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+
+        self._commit()
+
+    def save_signal(self, signal_data: dict):
+        if self.backend == "postgres":
+            upsert = """
+            INSERT INTO research_signals
+            (id, source, source_id, title, raw_text, authors, categories, url,
+             novelty_score, impact_score, metadata)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+            ON CONFLICT (id) DO UPDATE SET
+              source = EXCLUDED.source,
+              source_id = EXCLUDED.source_id,
+              title = EXCLUDED.title,
+              raw_text = EXCLUDED.raw_text,
+              authors = EXCLUDED.authors,
+              categories = EXCLUDED.categories,
+              url = EXCLUDED.url,
+              novelty_score = EXCLUDED.novelty_score,
+              impact_score = EXCLUDED.impact_score,
+              metadata = EXCLUDED.metadata
+            """
+        else:
+            upsert = """INSERT OR REPLACE INTO research_signals
+            (id, source, source_id, title, raw_text, authors, categories, url,
+             novelty_score, impact_score, metadata)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+
+        self._execute(
+            upsert,
+            (
+                signal_data.get("id", ""),
+                signal_data.get("source", ""),
+                signal_data.get("source_id", ""),
+                signal_data.get("title", ""),
+                signal_data.get("raw_text", ""),
+                json.dumps(signal_data.get("authors", [])),
+                json.dumps(signal_data.get("categories", [])),
+                signal_data.get("url", ""),
+                signal_data.get("novelty_score", 0),
+                signal_data.get("impact_score", 0),
+                json.dumps(signal_data.get("metadata", {})),
+            ),
+        )
+        self._commit()
+
+    def save_trend(self, trend_data: dict):
+        if self.backend == "postgres":
+            upsert = """
+            INSERT INTO trends
+            (id, rank, technique_name, description, emergence_score,
+             novelty_score, impact_score, mainstream_eta_months, confidence, data)
+            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+            ON CONFLICT (id) DO UPDATE SET
+              rank = EXCLUDED.rank,
+              technique_name = EXCLUDED.technique_name,
+              description = EXCLUDED.description,
+              emergence_score = EXCLUDED.emergence_score,
+              novelty_score = EXCLUDED.novelty_score,
+              impact_score = EXCLUDED.impact_score,
+              mainstream_eta_months = EXCLUDED.mainstream_eta_months,
+              confidence = EXCLUDED.confidence,
+              data = EXCLUDED.data,
+              updated_at = CURRENT_TIMESTAMP
+            """
+        else:
+            upsert = """INSERT OR REPLACE INTO trends
+            (id, rank, technique_name, description, emergence_score,
+             novelty_score, impact_score, mainstream_eta_months, confidence, data)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+
+        self._execute(
+            upsert,
+            (
+                trend_data.get("id", ""),
+                trend_data.get("rank", 0),
+                trend_data.get("technique_name", ""),
+                trend_data.get("description", ""),
+                trend_data.get("emergence_score", 0),
+                trend_data.get("novelty_score", 0),
+                trend_data.get("impact_score", 0),
+                trend_data.get("mainstream_eta_months", 12),
+                trend_data.get("confidence", 0),
+                json.dumps(trend_data),
+            ),
+        )
+        self._commit()
+
+    def get_signals_count(self) -> int:
+        cursor = self._execute("SELECT COUNT(*) FROM research_signals")
+        return cursor.fetchone()[0]
+
+    def get_signals_by_source(self, source: str) -> int:
+        ph = self._placeholder()
+        cursor = self._execute(
+            f"SELECT COUNT(*) FROM research_signals WHERE source = {ph}",
+            (source,),
+        )
+        return cursor.fetchone()[0]
+
+    def save_feedback(self, feedback_data: dict):
+        ph = self._placeholder()
+        self._execute(
+            f"""INSERT INTO user_feedback (id, target_id, target_type, action)
+            VALUES ({ph}, {ph}, {ph}, {ph})""",
+            (
+                feedback_data.get("id", ""),
+                feedback_data.get("target_id", ""),
+                feedback_data.get("target_type", ""),
+                feedback_data.get("action", ""),
+            ),
+        )
+        self._commit()
+
+    def save_pipeline(self, pipeline_data: dict):
+        """Persist generated/updated pipeline snapshot."""
+        if self.backend == "postgres":
+            upsert = """
+            INSERT INTO pipelines (id, technique_name, task_type, status, data)
+            VALUES (%s, %s, %s, %s, %s)
+            ON CONFLICT (id) DO UPDATE SET
+              technique_name = EXCLUDED.technique_name,
+              task_type = EXCLUDED.task_type,
+              status = EXCLUDED.status,
+              data = EXCLUDED.data
+            """
+        else:
+            upsert = """INSERT OR REPLACE INTO pipelines
+            (id, technique_name, task_type, status, data)
+            VALUES (?, ?, ?, ?, ?)"""
+        self._execute(
+            upsert,
+            (
+                pipeline_data.get("id", ""),
+                pipeline_data.get("technique_name", ""),
+                pipeline_data.get("task_type", ""),
+                pipeline_data.get("status", "generated"),
+                json.dumps(pipeline_data),
+            ),
+        )
+        self._commit()
+
+    def save_pipeline_run(self, run_data: dict):
+        """Persist pipeline run state."""
+        if self.backend == "postgres":
+            upsert = """
+            INSERT INTO pipeline_runs (run_id, pipeline_id, status, data)
+            VALUES (%s, %s, %s, %s)
+            ON CONFLICT (run_id) DO UPDATE SET
+              pipeline_id = EXCLUDED.pipeline_id,
+              status = EXCLUDED.status,
+              data = EXCLUDED.data,
+              updated_at = CURRENT_TIMESTAMP
+            """
+        else:
+            upsert = """INSERT OR REPLACE INTO pipeline_runs
+            (run_id, pipeline_id, status, data)
+            VALUES (?, ?, ?, ?)"""
+        self._execute(
+            upsert,
+            (
+                run_data.get("run_id", ""),
+                run_data.get("pipeline_id", ""),
+                run_data.get("status", "queued"),
+                json.dumps(run_data),
+            ),
+        )
+        self._commit()
+
+    def get_pipeline_runs(self, pipeline_id: str) -> list[dict]:
+        """Fetch all runs for a pipeline, newest first."""
+        ph = self._placeholder()
+        cursor = self._execute(
+            f"SELECT data FROM pipeline_runs WHERE pipeline_id = {ph} ORDER BY created_at DESC",
+            (pipeline_id,),
+        )
+        rows = cursor.fetchall()
+        out = []
+        for row in rows:
+            payload = row["data"] if isinstance(row, sqlite3.Row) else row[0]
+            out.append(json.loads(payload))
+        return out
+
+    def get_pipeline_run(self, run_id: str) -> Optional[dict]:
+        """Fetch one pipeline run by id."""
+        ph = self._placeholder()
+        cursor = self._execute(
+            f"SELECT data FROM pipeline_runs WHERE run_id = {ph}",
+            (run_id,),
+        )
+        row = cursor.fetchone()
+        if not row:
+            return None
+        payload = row["data"] if isinstance(row, sqlite3.Row) else row[0]
+        return json.loads(payload)
+
+    # ── Blueprints ──────────────────────────────────────────
+
+    def save_blueprint(self, blueprint_data: dict):
+        """Persist a generated blueprint."""
+        if self.backend == "postgres":
+            upsert = """
+            INSERT INTO blueprints (id, technique_name, trend_id, data)
+            VALUES (%s, %s, %s, %s)
+            ON CONFLICT (id) DO UPDATE SET
+              technique_name = EXCLUDED.technique_name,
+              trend_id = EXCLUDED.trend_id,
+              data = EXCLUDED.data
+            """
+        else:
+            upsert = """INSERT OR REPLACE INTO blueprints
+            (id, technique_name, trend_id, data)
+            VALUES (?, ?, ?, ?)"""
+        self._execute(
+            upsert,
+            (
+                blueprint_data.get("id", ""),
+                blueprint_data.get("technique_name", ""),
+                blueprint_data.get("trend_id", ""),
+                json.dumps(blueprint_data),
+            ),
+        )
+        self._commit()
+
+    def list_blueprints(self) -> list[dict]:
+        cursor = self._execute(
+            "SELECT data FROM blueprints ORDER BY created_at DESC"
+        )
+        rows = cursor.fetchall()
+        out = []
+        for row in rows:
+            payload = row["data"] if isinstance(row, sqlite3.Row) else row[0]
+            try:
+                out.append(json.loads(payload))
+            except Exception:
+                continue
+        return out
+
+    # ── Signals & Trends listing (for startup hydration) ────
+
+    def list_signals(self, limit: int = 1000) -> list[dict]:
+        """Return persisted research signals as plain dicts (no embedding).
+
+        Used at startup to rehydrate the in-memory vector store. Embeddings
+        are recomputed from ``raw_text`` because the column doesn't store
+        them.
+        """
+        ph = self._placeholder()
+        cursor = self._execute(
+            f"""SELECT id, source, source_id, title, raw_text, authors,
+                       categories, url, novelty_score, impact_score, metadata
+                FROM research_signals
+                ORDER BY created_at DESC
+                LIMIT {ph}""",
+            (int(limit),),
+        )
+        rows = cursor.fetchall()
+        out: list[dict] = []
+        for row in rows:
+            if isinstance(row, sqlite3.Row):
+                d = dict(row)
+            else:
+                cols = [
+                    "id", "source", "source_id", "title", "raw_text", "authors",
+                    "categories", "url", "novelty_score", "impact_score", "metadata",
+                ]
+                d = dict(zip(cols, row))
+            try:
+                d["authors"] = json.loads(d.get("authors") or "[]")
+            except Exception:
+                d["authors"] = []
+            try:
+                d["categories"] = json.loads(d.get("categories") or "[]")
+            except Exception:
+                d["categories"] = []
+            try:
+                d["metadata"] = json.loads(d.get("metadata") or "{}")
+            except Exception:
+                d["metadata"] = {}
+            out.append(d)
+        return out
+
+    def list_trends(self) -> list[dict]:
+        """Return persisted trends. Each row stores the full trend JSON in ``data``."""
+        cursor = self._execute(
+            "SELECT data FROM trends ORDER BY rank ASC"
+        )
+        rows = cursor.fetchall()
+        out: list[dict] = []
+        for row in rows:
+            payload = row["data"] if isinstance(row, sqlite3.Row) else row[0]
+            try:
+                out.append(json.loads(payload))
+            except Exception:
+                continue
+        return out
+
+    def list_pipelines(self) -> list[dict]:
+        cursor = self._execute(
+            "SELECT data FROM pipelines ORDER BY created_at DESC"
+        )
+        rows = cursor.fetchall()
+        out = []
+        for row in rows:
+            payload = row["data"] if isinstance(row, sqlite3.Row) else row[0]
+            try:
+                out.append(json.loads(payload))
+            except Exception:
+                continue
+        return out
+
+    # ── Telegram subscribers ────────────────────────────────
+
+    def ensure_telegram_subscribers_table(self):
+        """Idempotent ensure of the subscribers table for older deployments."""
+        cursor = self._conn.cursor()
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS telegram_subscribers (
+                chat_id BIGINT PRIMARY KEY,
+                username TEXT DEFAULT '',
+                created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+                updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        self._commit()
+
+    def upsert_telegram_subscriber(self, chat_id: int, username: str = ""):
+        if self.backend == "postgres":
+            sql = """
+            INSERT INTO telegram_subscribers (chat_id, username)
+            VALUES (%s, %s)
+            ON CONFLICT (chat_id) DO UPDATE SET
+              username = EXCLUDED.username,
+              updated_at = CURRENT_TIMESTAMP
+            """
+        else:
+            sql = """INSERT OR REPLACE INTO telegram_subscribers (chat_id, username)
+            VALUES (?, ?)"""
+        self._execute(sql, (int(chat_id), username or ""))
+        self._commit()
+
+    def delete_telegram_subscriber(self, chat_id: int):
+        ph = self._placeholder()
+        self._execute(
+            f"DELETE FROM telegram_subscribers WHERE chat_id = {ph}",
+            (int(chat_id),),
+        )
+        self._commit()
+
+    def list_telegram_subscriber_ids(self) -> list[int]:
+        cursor = self._execute("SELECT chat_id FROM telegram_subscribers ORDER BY created_at ASC")
+        rows = cursor.fetchall()
+        out = []
+        for row in rows:
+            cid = row["chat_id"] if isinstance(row, sqlite3.Row) else row[0]
+            try:
+                out.append(int(cid))
+            except Exception:
+                continue
+        return out
+
+    def close(self):
+        if self._conn:
+            self._conn.close()

delivery/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Delivery Package

delivery/api_routes.py

@@ -0,0 +1,781 @@
+"""REST API Routes — FastAPI endpoints for VectorMinds.
+
+Provides full programmatic access to all platform capabilities:
+trends, blueprints, pipelines, ingestion, stats, and vector map.
+Includes WebSocket for real-time dashboard updates.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Query, HTTPException, Header
+from pydantic import BaseModel
+
+from agents.ingestion_agent import IngestionAgent
+from agents.reasoning_agent import ReasoningAgent
+from agents.memory_agent import MemoryAgent
+from embeddings.engine import EmbeddingEngine
+from embeddings.vector_store import VectorStore
+from intelligence.blueprint_engine import BlueprintEngine
+from intelligence.experiment_designer import ExperimentDesigner
+from intelligence.pipeline_generator import PipelineGenerator
+from intelligence.pipeline_executor import PipelineExecutor
+from delivery.telegram_bot import TelegramBot
+from db.database import Database
+import config
+
+logger = logging.getLogger("vectorminds.api")
+
+router = APIRouter(prefix="/api")
+
+# ─── Global instances (set by main.py on startup) ────────────
+ingestion_agent: Optional[IngestionAgent] = None
+reasoning_agent: Optional[ReasoningAgent] = None
+memory_agent: Optional[MemoryAgent] = None
+blueprint_engine: Optional[BlueprintEngine] = None
+pipeline_generator: Optional[PipelineGenerator] = None
+pipeline_executor: Optional[PipelineExecutor] = None
+experiment_designer: Optional[ExperimentDesigner] = None
+telegram_bot: Optional[TelegramBot] = None
+embedding_engine: Optional[EmbeddingEngine] = None
+vector_store: Optional[VectorStore] = None
+database: Optional[Database] = None
+
+# WebSocket connections for live updates
+ws_connections: list[WebSocket] = []
+
+
+# ─── Request / Response Models ────────────────────────────────
+class IngestRequest(BaseModel):
+    source: str = "all"  # 'arxiv', 'github', or 'all'
+    category: Optional[str] = None  # e.g. 'cs.LG'
+    # When True, kick the ingestion run off in the background and return
+    # immediately with status="started". Lets mobile clients show a banner
+    # without holding open a 60–120s HTTP request that arXiv often pushes
+    # past their OkHttp read timeout.
+    background: bool = False
+
+
+# In-memory tracker for the most recent ingestion run. Keeps the UI honest
+# about what's happening on the server even when the call is fire-and-forget.
+_ingestion_status: dict = {
+    "state": "idle",  # 'idle' | 'running' | 'completed' | 'failed'
+    "started_at": None,
+    "finished_at": None,
+    "signals_ingested": 0,
+    "trends_updated": 0,
+    "error": None,
+}
+
+
+class BlueprintRequest(BaseModel):
+    trend_id: str
+    additional_context: str = ""
+
+
+class PipelineRequest(BaseModel):
+    technique_name: str
+    description: str = ""
+    task_type: Optional[str] = None
+
+class PipelineDatasetCandidatesRequest(BaseModel):
+    technique_name: str
+    description: str = ""
+    task_type: Optional[str] = None
+    top_k: int = 8
+
+
+class PipelineRunRequest(BaseModel):
+    timeout_seconds: int = config.PIPELINE_RUN_TIMEOUT_SECONDS
+    wait_for_completion: bool = False
+
+class ExperimentDesignRequest(BaseModel):
+    technique_name: str
+    brief: str = ""
+
+class DashboardPremiumContextResponse(BaseModel):
+    location: str
+    focus: str
+    next_meeting: str
+    author_name: str
+    papers_count: int
+    confidence: float
+    reasoning_points: list[str]
+    source_modes: dict
+
+
+class FeedbackRequest(BaseModel):
+    target_id: str
+    target_type: str = "trend"  # 'trend' or 'blueprint'
+    action: str = "upvote"  # 'upvote' or 'downvote'
+
+
+class SearchRequest(BaseModel):
+    query: str
+    top_k: int = 10
+    source_filter: Optional[str] = None
+
+
+def _assert_admin_api_key(x_api_key: Optional[str]):
+    if not config.API_ADMIN_KEY:
+        return
+    if x_api_key != config.API_ADMIN_KEY:
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+def _apply_run_snapshot_to_pipeline(pipeline, run: dict):
+    if run["status"] in ("completed", "failed", "timeout"):
+        pipeline.status = run["status"]
+    else:
+        pipeline.status = "training"
+    metrics = dict(pipeline.metrics or {})
+    metrics["last_run"] = {
+        "run_id": run["run_id"],
+        "status": run["status"],
+        "started_at": run.get("started_at"),
+        "finished_at": run.get("finished_at"),
+        "exit_code": run.get("exit_code"),
+        "duration_seconds": run.get("duration_seconds"),
+        "log_path": run.get("log_path"),
+        "artifacts_dir": run.get("artifacts_dir"),
+        "retry_count": run.get("retry_count", 0),
+        "max_retries": run.get("max_retries", 0),
+    }
+    pipeline.metrics = metrics
+
+
+# ─── Broadcast helper ────────────────────────────────────────
+async def broadcast_ws(event_type: str, data: dict):
+    """Send real-time update to all connected WebSocket clients."""
+    message = json.dumps({"type": event_type, "data": data, "timestamp": datetime.now(timezone.utc).isoformat()})
+    disconnected = []
+    for ws in ws_connections:
+        try:
+            await ws.send_text(message)
+        except Exception:
+            disconnected.append(ws)
+    for ws in disconnected:
+        ws_connections.remove(ws)
+
+
+# ─── Endpoints ────────────────────────────────────────────────
+
+@router.get("/health")
+async def health_check():
+    """Platform health check."""
+    agents_health = {}
+    if ingestion_agent:
+        agents_health["ingestion"] = ingestion_agent.get_health()
+    if reasoning_agent:
+        agents_health["reasoning"] = reasoning_agent.get_health()
+    if memory_agent:
+        agents_health["memory"] = memory_agent.get_health()
+
+    return {
+        "status": "healthy",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "agents": agents_health,
+        "vector_store_count": vector_store.get_collection_count() if vector_store else 0,
+        "infra": {
+            "event_bus_backend": config.MESSAGE_BUS_BACKEND,
+            "state_store_backend": config.STATE_STORE_BACKEND,
+            "db_backend": config.DB_BACKEND,
+            "vector_store_backend": "qdrant_in_memory" if not config.QDRANT_HOST else "qdrant_remote",
+        },
+    }
+
+
+@router.get("/stats")
+async def get_stats():
+    """Get platform statistics for dashboard."""
+    vs_count = vector_store.get_collection_count() if vector_store else 0
+    db_papers = database.get_signals_by_source("arxiv") if database else 0
+    db_repos = database.get_signals_by_source("github") if database else 0
+
+    # Get novelty distribution from stored signals
+    payloads = vector_store.get_all_payloads(limit=200) if vector_store else []
+    novelty_scores = [p.get("novelty_score", 0) for p in payloads]
+
+    return {
+        "total_signals": vs_count,
+        "total_papers": db_papers,
+        "total_github_repos": db_repos,
+        "active_trends": len(reasoning_agent.trends) if reasoning_agent else 0,
+        "blueprints_generated": len(blueprint_engine.generated_blueprints) if blueprint_engine else 0,
+        "pipelines_launched": len(pipeline_generator.generated_pipelines) if pipeline_generator else 0,
+        "avg_novelty_score": round(sum(novelty_scores) / max(len(novelty_scores), 1), 3),
+        "novelty_distribution": novelty_scores[:100],
+        "agents_status": {
+            "ingestion": ingestion_agent.status if ingestion_agent else "offline",
+            "reasoning": reasoning_agent.status if reasoning_agent else "offline",
+            "memory": memory_agent.status if memory_agent else "offline",
+        },
+        "source_modes": {
+            "patents_real": config.ENABLE_PATENTS_REAL,
+            "startups_real": config.ENABLE_STARTUPS_REAL,
+            "social_real": config.ENABLE_SOCIAL_REAL,
+            "blog_real": config.ENABLE_BLOG_REAL,
+            "allow_simulated_fallback": config.ALLOW_SIMULATED_SOURCES,
+        },
+        "telegram": telegram_bot.get_stats() if telegram_bot else {},
+        "last_updated": datetime.now(timezone.utc).isoformat(),
+    }
+
+@router.get("/dashboard/premium-context", response_model=DashboardPremiumContextResponse)
+async def get_dashboard_premium_context():
+    """Backend-derived context for premium dashboard panels."""
+    trends = []
+    if reasoning_agent and reasoning_agent.trends:
+        trends = sorted(
+            reasoning_agent.trends.values(),
+            key=lambda t: t.emergence_score,
+            reverse=True,
+        )
+    elif reasoning_agent:
+        trends = await reasoning_agent.analyze_trends()
+
+    top = trends[0] if trends else None
+    technique = top.technique_name if top else "Autonomous Research Discovery"
+    papers = top.paper_count if top else 0
+    confidence = float(top.confidence if top else 0.74)
+
+    location = (
+        "Distributed Lab (Cloud + Device)"
+        if config.STATE_STORE_BACKEND == "redis"
+        else "Local Research Runtime"
+    )
+    next_meeting = f"Trend Review: {technique} ({'6' if top and top.mainstream_eta_months <= 6 else '12/24'} month horizon)"
+    reasoning_points = [
+        f"Top ranked technique is '{technique}' from live trend analysis",
+        f"Cross-source evidence includes {papers} papers and {top.github_stars if top else 0} GitHub stars",
+        f"Current backend mode: DB={config.DB_BACKEND}, Bus={config.MESSAGE_BUS_BACKEND}, State={config.STATE_STORE_BACKEND}",
+    ]
+    source_modes = {
+        "patents_real": config.ENABLE_PATENTS_REAL,
+        "startups_real": config.ENABLE_STARTUPS_REAL,
+        "social_real": config.ENABLE_SOCIAL_REAL,
+        "blog_real": config.ENABLE_BLOG_REAL,
+        "allow_simulated_fallback": config.ALLOW_SIMULATED_SOURCES,
+    }
+    return DashboardPremiumContextResponse(
+        location=location,
+        focus=technique,
+        next_meeting=next_meeting,
+        author_name="Top Signal Cluster",
+        papers_count=papers,
+        confidence=confidence,
+        reasoning_points=reasoning_points,
+        source_modes=source_modes,
+    )
+
+
+async def _run_ingestion_pipeline(req: IngestRequest) -> dict:
+    """Execute the full ingest → analyze → broadcast pipeline.
+
+    Returns a result dict regardless of whether it was awaited inline or
+    scheduled as a background task. Updates the shared `_ingestion_status`
+    so polling clients can observe progress.
+    """
+    global _ingestion_status
+    _ingestion_status = {
+        "state": "running",
+        "started_at": datetime.now(timezone.utc).isoformat(),
+        "finished_at": None,
+        "signals_ingested": 0,
+        "trends_updated": 0,
+        "error": None,
+    }
+
+    try:
+        signals = await ingestion_agent.run_ingestion(
+            source=req.source, category=req.category
+        )
+
+        if database:
+            for s in signals:
+                database.save_signal(s.model_dump(mode="json"))
+
+        trends_count = 0
+        if reasoning_agent:
+            trends = await reasoning_agent.analyze_trends()
+            trends_count = len(reasoning_agent.trends)
+            if database:
+                for t in trends:
+                    database.save_trend(t.model_dump(mode="json"))
+            if telegram_bot:
+                for t in trends[:3]:
+                    if t.impact_score >= config.IMPACT_HIGH_THRESHOLD:
+                        await telegram_bot.send_trend_alert(
+                            technique=t.technique_name,
+                            score=t.emergence_score,
+                            eta=t.mainstream_eta_months,
+                        )
+
+        await broadcast_ws("ingestion_complete", {
+            "count": len(signals),
+            "source": req.source,
+            "signals": [
+                {
+                    "id": s.id,
+                    "source": s.source.value,
+                    "title": s.title,
+                    "novelty_score": s.novelty_score,
+                    "url": s.url,
+                }
+                for s in signals[:20]
+            ],
+        })
+
+        if telegram_bot:
+            arxiv_count = sum(1 for s in signals if s.source.value == "arxiv")
+            github_count = sum(1 for s in signals if s.source.value == "github")
+            await telegram_bot.send_ingestion_summary(arxiv_count, github_count)
+
+        result = {
+            "status": "success",
+            "signals_ingested": len(signals),
+            "trends_updated": trends_count,
+        }
+        _ingestion_status = {
+            "state": "completed",
+            "started_at": _ingestion_status["started_at"],
+            "finished_at": datetime.now(timezone.utc).isoformat(),
+            "signals_ingested": len(signals),
+            "trends_updated": trends_count,
+            "error": None,
+        }
+        return result
+    except Exception as e:
+        logger.exception("Ingestion pipeline failed")
+        _ingestion_status = {
+            "state": "failed",
+            "started_at": _ingestion_status["started_at"],
+            "finished_at": datetime.now(timezone.utc).isoformat(),
+            "signals_ingested": 0,
+            "trends_updated": 0,
+            "error": str(e),
+        }
+        raise
+
+
+@router.post("/ingest")
+async def trigger_ingestion(req: IngestRequest):
+    """Trigger a manual ingestion run.
+
+    With `background=True`, schedules the run on the event loop and returns
+    immediately with status="started" so the mobile UI can show a banner and
+    poll `/api/ingest/status` for completion. Otherwise runs inline (used by
+    integration tests and curl smoke tests).
+    """
+    if not ingestion_agent:
+        raise HTTPException(status_code=503, detail="Ingestion agent not ready")
+
+    if req.background:
+        if _ingestion_status.get("state") == "running":
+            return {
+                "status": "already_running",
+                "started_at": _ingestion_status.get("started_at"),
+            }
+        asyncio.create_task(_run_ingestion_pipeline(req))
+        return {
+            "status": "started",
+            "message": "Ingestion is running in the background. Poll /api/ingest/status.",
+        }
+
+    return await _run_ingestion_pipeline(req)
+
+
+@router.get("/ingest/status")
+async def get_ingestion_status():
+    """Return the state of the last/in-flight ingestion run."""
+    return _ingestion_status
+
+
+@router.get("/trends")
+async def get_trends(limit: int = Query(default=20, le=100)):
+    """Get the trend leaderboard."""
+    if not reasoning_agent or not reasoning_agent.trends:
+        # Run analysis if no trends exist yet
+        if reasoning_agent:
+            trends = await reasoning_agent.analyze_trends()
+        else:
+            return {"trends": [], "count": 0}
+    else:
+        trends = sorted(
+            reasoning_agent.trends.values(),
+            key=lambda t: t.emergence_score,
+            reverse=True,
+        )
+
+    trend_list = [t.model_dump(mode="json") for t in trends[:limit]]
+    return {"trends": trend_list, "count": len(trend_list)}
+
+
+@router.get("/trends/{trend_id}")
+async def get_trend_detail(
+    trend_id: str,
+    include_brief: bool = Query(
+        default=False,
+        description="When true, runs Gemini to generate technical_brief (slow). "
+        "Omit or false for instant scores + description.",
+    ),
+):
+    """Get detailed view of a specific trend."""
+    if not reasoning_agent:
+        raise HTTPException(status_code=503, detail="Reasoning agent not ready")
+
+    trend = reasoning_agent.trends.get(trend_id)
+    if not trend:
+        raise HTTPException(status_code=404, detail="Trend not found")
+
+    result = trend.model_dump(mode="json")
+    if include_brief:
+        brief = await reasoning_agent.generate_technical_brief(
+            trend.technique_name, trend.description
+        )
+        result["technical_brief"] = brief
+    else:
+        result["technical_brief"] = None
+    return result
+
+
+@router.post("/blueprints/generate")
+async def generate_blueprint(req: BlueprintRequest):
+    """Generate a product blueprint for a trend."""
+    if not blueprint_engine or not reasoning_agent:
+        raise HTTPException(status_code=503, detail="Services not ready")
+
+    trend = reasoning_agent.trends.get(req.trend_id)
+    if not trend:
+        raise HTTPException(status_code=404, detail="Trend not found")
+
+    blueprint = await blueprint_engine.generate_blueprint(
+        trend, req.additional_context
+    )
+
+    # Store in memory agent
+    if memory_agent:
+        memory_agent.store_blueprint(blueprint.id, blueprint.model_dump(mode="json"))
+    if database:
+        try:
+            database.save_blueprint(blueprint.model_dump(mode="json"))
+        except Exception as e:
+            logger.warning("blueprint persist failed: %s", e)
+
+    await broadcast_ws("blueprint_generated", {
+        "id": blueprint.id,
+        "technique": blueprint.technique_name,
+    })
+
+    return blueprint.model_dump(mode="json")
+
+
+@router.get("/blueprints")
+async def list_blueprints():
+    """List all generated blueprints."""
+    if not blueprint_engine:
+        return {"blueprints": [], "count": 0}
+    bps = [b.model_dump(mode="json") for b in blueprint_engine.list_blueprints()]
+    return {"blueprints": bps, "count": len(bps)}
+
+
+@router.get("/blueprints/{blueprint_id}")
+async def get_blueprint(blueprint_id: str):
+    """Get a specific blueprint."""
+    if not blueprint_engine:
+        raise HTTPException(status_code=503, detail="Blueprint engine not ready")
+    bp = blueprint_engine.get_blueprint(blueprint_id)
+    if not bp:
+        raise HTTPException(status_code=404, detail="Blueprint not found")
+    return bp.model_dump(mode="json")
+
+
+@router.post("/pipelines/generate")
+async def generate_pipeline(req: PipelineRequest):
+    """Generate an ML training pipeline."""
+    if not pipeline_generator:
+        raise HTTPException(status_code=503, detail="Pipeline generator not ready")
+
+    pipeline = pipeline_generator.generate_pipeline(
+        technique_name=req.technique_name,
+        description=req.description,
+        task_type=req.task_type,
+    )
+
+    if memory_agent:
+        memory_agent.store_pipeline(pipeline.model_dump(mode="json"))
+    if database:
+        database.save_pipeline(pipeline.model_dump(mode="json"))
+
+    await broadcast_ws("pipeline_generated", {
+        "id": pipeline.id,
+        "technique": pipeline.technique_name,
+        "task_type": pipeline.task_type,
+    })
+    if telegram_bot:
+        await telegram_bot.send_pipeline_complete(
+            technique=pipeline.technique_name,
+            task_type=pipeline.task_type,
+            metrics=pipeline.metrics,
+            colab_url=pipeline.colab_url,
+        )
+
+    return pipeline.model_dump(mode="json")
+
+
+@router.post("/pipelines/{pipeline_id}/run")
+async def run_pipeline(pipeline_id: str, req: PipelineRunRequest, x_api_key: Optional[str] = Header(default=None)):
+    """Execute a generated pipeline script."""
+    _assert_admin_api_key(x_api_key)
+    if not pipeline_generator or not pipeline_executor:
+        raise HTTPException(status_code=503, detail="Pipeline services not ready")
+
+    pipeline = pipeline_generator.get_pipeline(pipeline_id)
+    if not pipeline:
+        raise HTTPException(status_code=404, detail="Pipeline not found")
+
+    timeout = min(max(req.timeout_seconds, 30), 7200)
+    pipeline.status = "training"
+    pipeline_generator.update_pipeline(pipeline)
+    if database:
+        database.save_pipeline(pipeline.model_dump(mode="json"))
+
+    if req.wait_for_completion:
+        run = await pipeline_executor.execute_pipeline(pipeline, timeout_seconds=timeout)
+    else:
+        run = pipeline_executor.execute_pipeline_async(pipeline, timeout_seconds=timeout)
+
+    _apply_run_snapshot_to_pipeline(pipeline, run)
+    pipeline_generator.update_pipeline(pipeline)
+    if database:
+        database.save_pipeline(pipeline.model_dump(mode="json"))
+        database.save_pipeline_run(run)
+
+    await broadcast_ws("pipeline_run_started", {
+        "pipeline_id": pipeline.id,
+        "run_id": run["run_id"],
+        "status": run["status"],
+        "technique": pipeline.technique_name,
+    })
+
+    return {
+        "status": "accepted" if run["status"] in ("queued", "running") else "finished",
+        "pipeline": pipeline.model_dump(mode="json"),
+        "run": run,
+    }
+
+@router.post("/pipelines/dataset-candidates")
+async def pipeline_dataset_candidates(req: PipelineDatasetCandidatesRequest):
+    """Preview ranked dataset candidates before pipeline generation."""
+    if not pipeline_generator:
+        raise HTTPException(status_code=503, detail="Pipeline generator not ready")
+    candidates = pipeline_generator.dataset_candidates(
+        technique_name=req.technique_name,
+        description=req.description,
+        task_type=req.task_type,
+        top_k=min(max(req.top_k, 1), 20),
+    )
+    return {"candidates": candidates, "count": len(candidates)}
+
+@router.post("/experiments/design")
+async def design_experiment(req: ExperimentDesignRequest):
+    """Generate a minimal viable experiment design for a technique."""
+    if not experiment_designer:
+        raise HTTPException(status_code=503, detail="Experiment designer not ready")
+    exp = await experiment_designer.design_experiment(
+        technique_name=req.technique_name,
+        brief=req.brief,
+    )
+    return exp
+
+
+@router.get("/pipelines")
+async def list_pipelines():
+    """List all generated pipelines."""
+    if not pipeline_generator:
+        return {"pipelines": [], "count": 0}
+    pls = [p.model_dump(mode="json") for p in pipeline_generator.list_pipelines()]
+    return {"pipelines": pls, "count": len(pls)}
+
+
+@router.get("/pipelines/{pipeline_id}")
+async def get_pipeline(pipeline_id: str):
+    """Get a specific pipeline."""
+    if not pipeline_generator:
+        raise HTTPException(status_code=503, detail="Pipeline generator not ready")
+    pl = pipeline_generator.get_pipeline(pipeline_id)
+    if not pl:
+        raise HTTPException(status_code=404, detail="Pipeline not found")
+    return pl.model_dump(mode="json")
+
+
+@router.get("/pipelines/{pipeline_id}/runs")
+async def list_pipeline_runs(pipeline_id: str):
+    """List runs for a pipeline."""
+    if not pipeline_generator or not pipeline_executor:
+        raise HTTPException(status_code=503, detail="Pipeline services not ready")
+    pipeline = pipeline_generator.get_pipeline(pipeline_id)
+    if not pipeline:
+        raise HTTPException(status_code=404, detail="Pipeline not found")
+    runs = pipeline_executor.list_runs(pipeline_id)
+    if not runs and database:
+        runs = database.get_pipeline_runs(pipeline_id)
+    return {"pipeline_id": pipeline_id, "runs": runs, "count": len(runs)}
+
+
+@router.get("/pipelines/{pipeline_id}/runs/{run_id}")
+async def get_pipeline_run(pipeline_id: str, run_id: str):
+    """Get run status for a specific pipeline run."""
+    if not pipeline_generator or not pipeline_executor:
+        raise HTTPException(status_code=503, detail="Pipeline services not ready")
+    pipeline = pipeline_generator.get_pipeline(pipeline_id)
+    if not pipeline:
+        raise HTTPException(status_code=404, detail="Pipeline not found")
+
+    run = pipeline_executor.get_run(pipeline_id, run_id)
+    if not run and database:
+        run = database.get_pipeline_run(run_id)
+    if not run:
+        raise HTTPException(status_code=404, detail="Run not found")
+
+    _apply_run_snapshot_to_pipeline(pipeline, run)
+    pipeline_generator.update_pipeline(pipeline)
+    if database:
+        database.save_pipeline(pipeline.model_dump(mode="json"))
+        database.save_pipeline_run(run)
+
+    return run
+
+
+@router.get("/pipelines/{pipeline_id}/runs/{run_id}/log")
+async def get_pipeline_run_log(pipeline_id: str, run_id: str, tail_lines: int = Query(default=200, ge=10, le=2000)):
+    """Read latest log lines for a pipeline run."""
+    if not pipeline_generator or not pipeline_executor:
+        raise HTTPException(status_code=503, detail="Pipeline services not ready")
+    pipeline = pipeline_generator.get_pipeline(pipeline_id)
+    if not pipeline:
+        raise HTTPException(status_code=404, detail="Pipeline not found")
+    run = pipeline_executor.get_run(pipeline_id, run_id)
+    if not run:
+        raise HTTPException(status_code=404, detail="Run not found")
+
+    try:
+        with open(run["log_path"], "r", encoding="utf-8") as f:
+            lines = f.readlines()
+    except FileNotFoundError:
+        lines = []
+
+    sliced = lines[-tail_lines:]
+    return {
+        "pipeline_id": pipeline_id,
+        "run_id": run_id,
+        "status": run["status"],
+        "line_count": len(sliced),
+        "log_tail": "".join(sliced),
+    }
+
+
+@router.post("/search")
+async def semantic_search(req: SearchRequest):
+    """Semantic search across all research signals."""
+    if not embedding_engine or not vector_store:
+        raise HTTPException(status_code=503, detail="Search not ready")
+
+    query_embedding = embedding_engine.embed_text(req.query)
+    results = vector_store.search(
+        query_vector=query_embedding,
+        top_k=req.top_k,
+        source_filter=req.source_filter,
+    )
+    return {"results": results, "count": len(results), "query": req.query}
+
+
+@router.get("/vector-map")
+async def get_vector_map(limit: int = Query(default=200, le=500)):
+    """Get 2D projection of vector space for visualization."""
+    if not vector_store:
+        return {"points": [], "count": 0}
+
+    vectors, payloads = vector_store.get_vectors_for_projection(limit=limit)
+    if not vectors:
+        return {"points": [], "count": 0}
+
+    # Simple 2D projection using PCA (fast for demo)
+    import numpy as np
+    from sklearn.decomposition import PCA
+
+    vecs = np.array(vectors)
+    if len(vecs) < 2:
+        return {"points": [], "count": 0}
+
+    n_components = min(2, len(vecs), vecs.shape[1])
+    pca = PCA(n_components=n_components)
+    projected = pca.fit_transform(vecs)
+
+    points = []
+    for i, (coords, payload) in enumerate(zip(projected, payloads)):
+        points.append({
+            "x": float(coords[0]) if len(coords) > 0 else 0,
+            "y": float(coords[1]) if len(coords) > 1 else 0,
+            "title": payload.get("title", ""),
+            "source": payload.get("source", ""),
+            "novelty_score": payload.get("novelty_score", 0),
+            "categories": payload.get("categories", []),
+        })
+
+    return {
+        "points": points,
+        "count": len(points),
+        "explained_variance": pca.explained_variance_ratio_.tolist(),
+    }
+
+
+@router.post("/feedback")
+async def submit_feedback(req: FeedbackRequest):
+    """Submit user feedback (upvote/downvote) on a prediction."""
+    import uuid
+
+    feedback_data = {
+        "id": str(uuid.uuid4()),
+        "target_id": req.target_id,
+        "target_type": req.target_type,
+        "action": req.action,
+    }
+
+    if database:
+        database.save_feedback(feedback_data)
+
+    if memory_agent:
+        await memory_agent.bus.publish_simple(
+            "delivery.feedback", "api", feedback_data
+        )
+
+    return {"status": "recorded", "feedback": feedback_data}
+
+
+# ─── WebSocket for Live Updates ───────────────────────────────
+@router.websocket("/ws/live")
+async def websocket_live(ws: WebSocket):
+    """WebSocket endpoint for real-time dashboard updates."""
+    await ws.accept()
+    ws_connections.append(ws)
+    logger.info(f"WebSocket client connected (total: {len(ws_connections)})")
+
+    try:
+        while True:
+            # Keep connection alive, handle incoming messages
+            data = await ws.receive_text()
+            # Echo or handle client commands
+            if data == "ping":
+                await ws.send_text(json.dumps({"type": "pong"}))
+    except WebSocketDisconnect:
+        ws_connections.remove(ws)
+        logger.info(f"WebSocket client disconnected (total: {len(ws_connections)})")
+    except Exception:
+        if ws in ws_connections:
+            ws_connections.remove(ws)

delivery/colab_publisher.py

@@ -0,0 +1,98 @@
+"""Publish generated VectorMinds notebooks to a real Google Colab URL.
+
+Strategy: create or update a public GitHub Gist (using the existing
+``GITHUB_TOKEN``) holding the ``.ipynb`` file. Colab can open any public Gist via
+``https://colab.research.google.com/gist/<owner>/<gist_id>/<filename>.ipynb``.
+
+If no token is available or GitHub is unreachable, ``publish_notebook`` returns
+``None`` so the pipeline still works with an in-memory notebook.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+import httpx
+
+import config
+
+logger = logging.getLogger("vectorminds.colab")
+
+GITHUB_API = "https://api.github.com"
+
+
+def _auth_headers() -> dict:
+    h = {"Accept": "application/vnd.github+json", "User-Agent": "VectorMinds/1.0"}
+    if config.GITHUB_TOKEN:
+        h["Authorization"] = f"Bearer {config.GITHUB_TOKEN}"
+    return h
+
+
+def _resolve_owner() -> Optional[str]:
+    """Return the GitHub username for the configured ``GITHUB_TOKEN``."""
+    if not config.GITHUB_TOKEN:
+        return None
+    try:
+        with httpx.Client(timeout=10.0, headers=_auth_headers()) as client:
+            r = client.get(f"{GITHUB_API}/user")
+            if r.status_code != 200:
+                logger.warning("GitHub /user returned %s: %s", r.status_code, r.text[:200])
+                return None
+            return (r.json() or {}).get("login")
+    except Exception as e:
+        logger.warning("GitHub /user failed: %s", e)
+        return None
+
+
+def publish_notebook(
+    notebook_payload: dict,
+    filename: str,
+    description: str = "",
+    public: bool = True,
+) -> Optional[dict]:
+    """Create a Gist holding a single ``.ipynb`` and return ``{owner, gist_id, colab_url, gist_url}``.
+
+    Returns ``None`` if publishing is not possible (no token, network failure).
+    """
+    if not config.GITHUB_TOKEN:
+        logger.info("GitHub token not configured; skipping Colab gist publish.")
+        return None
+    import json
+
+    body = {
+        "description": description or "VectorMinds generated training pipeline",
+        "public": bool(public),
+        "files": {filename: {"content": json.dumps(notebook_payload, ensure_ascii=False, indent=2)}},
+    }
+    try:
+        with httpx.Client(timeout=20.0, headers=_auth_headers()) as client:
+            r = client.post(f"{GITHUB_API}/gists", json=body)
+            if r.status_code not in (200, 201):
+                logger.warning("Gist create failed %s: %s", r.status_code, r.text[:300])
+                return None
+            data = r.json()
+    except Exception as e:
+        logger.warning("Gist create exception: %s", e)
+        return None
+
+    gist_id = data.get("id")
+    owner_login = (data.get("owner") or {}).get("login") or _resolve_owner()
+    gist_url = data.get("html_url") or (
+        f"https://gist.github.com/{owner_login}/{gist_id}" if (owner_login and gist_id) else ""
+    )
+    if not gist_id:
+        return None
+    if not owner_login:
+        owner_login = _resolve_owner() or "anonymous"
+    colab_url = (
+        f"https://colab.research.google.com/gist/{owner_login}/{gist_id}/{filename}"
+    )
+    logger.info("Published Colab gist %s for %s", gist_id, filename)
+    return {
+        "owner": owner_login,
+        "gist_id": gist_id,
+        "gist_url": gist_url,
+        "colab_url": colab_url,
+        "filename": filename,
+    }

delivery/telegram_bot.py

@@ -0,0 +1,398 @@
+"""Production Telegram bot for VectorMinds.
+
+Real two-way bot built on ``python-telegram-bot`` 21:
+- subscribers register with ``/start`` (chat ids persisted in Postgres/SQLite)
+- alerts (trend, ingestion summary, pipeline complete) broadcast to every subscriber
+- live commands: ``/start /help /status /trends /pipelines /unsubscribe``
+
+The bot has no mock fallback: when ``TELEGRAM_BOT_TOKEN`` is unset the module logs a
+clear warning and ``send_*`` methods become no-ops returning ``False``. When a token
+is set, every send is a real Telegram API call and failures are returned as ``False``.
+
+The bot lifecycle (``start_polling`` / ``stop``) is managed from ``main.lifespan`` so
+polling runs alongside FastAPI without blocking the event loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import Awaitable, Callable, Optional
+
+from telegram import Update
+from telegram.constants import ParseMode
+from telegram.error import TelegramError
+from telegram.ext import (
+    Application,
+    ApplicationBuilder,
+    CommandHandler,
+    ContextTypes,
+)
+
+import config
+
+logger = logging.getLogger("vectorminds.telegram")
+
+
+WELCOME = (
+    "<b>VectorMinds</b> - GenAI Research Intelligence\n"
+    "You are subscribed to live alerts.\n\n"
+    "Commands:\n"
+    "/help - show this help\n"
+    "/status - platform stats\n"
+    "/trends - top emerging techniques\n"
+    "/pipelines - recent ML pipelines\n"
+    "/unsubscribe - stop receiving alerts"
+)
+
+HELP_MESSAGE = (
+    "<b>VectorMinds Bot</b>\n"
+    "/start - subscribe and show this menu\n"
+    "/help - show this help\n"
+    "/status - platform stats (signals, trends, pipelines)\n"
+    "/trends - top 5 ranked techniques\n"
+    "/pipelines - recent ML pipelines\n"
+    "/unsubscribe - stop receiving alerts"
+)
+
+
+class TelegramBot:
+    """Production Telegram delivery and command surface for VectorMinds."""
+
+    def __init__(self, token: str = "", database=None):
+        self.token = (token or config.TELEGRAM_BOT_TOKEN or "").strip()
+        self.database = database
+        self.enabled: bool = bool(self.token)
+        self._sent_count = 0
+        self._failed_count = 0
+        self._app: Optional[Application] = None
+        self._polling_started = False
+        # Optional callable returning a fresh ``stats`` dict for ``/status``.
+        self._stats_provider: Optional[Callable[[], Awaitable[dict]]] = None
+        # Optional callable returning ``list[dict]`` of trends for ``/trends``.
+        self._trends_provider: Optional[Callable[[int], Awaitable[list[dict]]]] = None
+        # Optional callable returning ``list[dict]`` of pipelines for ``/pipelines``.
+        self._pipelines_provider: Optional[Callable[[int], Awaitable[list[dict]]]] = None
+        if not self.enabled:
+            logger.warning(
+                "Telegram bot disabled - set TELEGRAM_BOT_TOKEN to enable real delivery"
+            )
+
+    # ── lifecycle ─────────────────────────────────────────────
+
+    async def start_polling(self) -> None:
+        """Start the long-polling task. Safe to call once; idempotent on retry."""
+        if not self.enabled:
+            return
+        if self._polling_started:
+            return
+        try:
+            self._app = (
+                ApplicationBuilder()
+                .token(self.token)
+                .concurrent_updates(True)
+                .build()
+            )
+            self._app.add_handler(CommandHandler("start", self._cmd_start))
+            self._app.add_handler(CommandHandler("help", self._cmd_help))
+            self._app.add_handler(CommandHandler("status", self._cmd_status))
+            self._app.add_handler(CommandHandler("trends", self._cmd_trends))
+            self._app.add_handler(CommandHandler("pipelines", self._cmd_pipelines))
+            self._app.add_handler(CommandHandler("unsubscribe", self._cmd_unsubscribe))
+
+            await self._app.initialize()
+            await self._app.start()
+            await self._app.updater.start_polling(drop_pending_updates=False)
+            self._polling_started = True
+            me = await self._app.bot.get_me()
+            count = self._subscriber_count()
+            logger.info(
+                "Telegram bot @%s online (subscribers=%s)", me.username, count
+            )
+        except Exception as e:
+            logger.error("Telegram polling failed to start: %s", e)
+            self.enabled = False
+            self._app = None
+            self._polling_started = False
+
+    async def stop_polling(self) -> None:
+        """Cleanly stop the polling task. Safe to call multiple times."""
+        app = self._app
+        if not app:
+            return
+        try:
+            if app.updater and app.updater.running:
+                await app.updater.stop()
+            if app.running:
+                await app.stop()
+            await app.shutdown()
+        except Exception as e:
+            logger.warning("Telegram bot stop encountered: %s", e)
+        finally:
+            self._app = None
+            self._polling_started = False
+
+    def attach_providers(
+        self,
+        stats: Optional[Callable[[], Awaitable[dict]]] = None,
+        trends: Optional[Callable[[int], Awaitable[list[dict]]]] = None,
+        pipelines: Optional[Callable[[int], Awaitable[list[dict]]]] = None,
+    ) -> None:
+        """Wire callables that resolve dynamic data for ``/status``, ``/trends`` and ``/pipelines``."""
+        if stats is not None:
+            self._stats_provider = stats
+        if trends is not None:
+            self._trends_provider = trends
+        if pipelines is not None:
+            self._pipelines_provider = pipelines
+
+    # ── command handlers ─────────────────────────────────────
+
+    async def _cmd_start(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        user = update.effective_user
+        if not chat:
+            return
+        self._upsert_subscriber(chat.id, user.username if user else None)
+        await context.bot.send_message(
+            chat_id=chat.id, text=WELCOME, parse_mode=ParseMode.HTML
+        )
+
+    async def _cmd_help(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        if not chat:
+            return
+        await context.bot.send_message(
+            chat_id=chat.id, text=HELP_MESSAGE, parse_mode=ParseMode.HTML
+        )
+
+    async def _cmd_status(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        if not chat:
+            return
+        if not self._stats_provider:
+            await context.bot.send_message(
+                chat_id=chat.id, text="Stats not available right now."
+            )
+            return
+        try:
+            data = await self._stats_provider()
+        except Exception as e:
+            logger.warning("status provider failed: %s", e)
+            await context.bot.send_message(chat_id=chat.id, text="Stats are temporarily unavailable.")
+            return
+        agents = data.get("agents_status") or {}
+        text = (
+            "<b>VectorMinds status</b>\n"
+            f"Total signals: <b>{data.get('total_signals', 0)}</b>\n"
+            f"Active trends: <b>{data.get('active_trends', 0)}</b>\n"
+            f"Blueprints: <b>{data.get('blueprints_generated', 0)}</b>\n"
+            f"Pipelines: <b>{data.get('pipelines_launched', 0)}</b>\n"
+            f"Avg novelty: <b>{data.get('avg_novelty_score', 0)}</b>\n"
+            f"Agents: ingestion=<b>{agents.get('ingestion', '?')}</b> "
+            f"reasoning=<b>{agents.get('reasoning', '?')}</b> "
+            f"memory=<b>{agents.get('memory', '?')}</b>"
+        )
+        await context.bot.send_message(chat_id=chat.id, text=text, parse_mode=ParseMode.HTML)
+
+    async def _cmd_trends(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        if not chat:
+            return
+        if not self._trends_provider:
+            await context.bot.send_message(chat_id=chat.id, text="Trend service is offline.")
+            return
+        try:
+            trends = await self._trends_provider(5)
+        except Exception as e:
+            logger.warning("trend provider failed: %s", e)
+            await context.bot.send_message(chat_id=chat.id, text="Trends temporarily unavailable.")
+            return
+        if not trends:
+            await context.bot.send_message(
+                chat_id=chat.id,
+                text="No trends yet. Trigger an ingestion run from the API and try again.",
+            )
+            return
+        lines = ["<b>Top Trends</b>"]
+        for i, t in enumerate(trends, start=1):
+            lines.append(
+                f"{i}. <b>{t.get('technique_name', '?')}</b> | "
+                f"emergence={float(t.get('emergence_score', 0)):.2f} | "
+                f"ETA {t.get('mainstream_eta_months', '?')}mo"
+            )
+        await context.bot.send_message(
+            chat_id=chat.id, text="\n".join(lines), parse_mode=ParseMode.HTML
+        )
+
+    async def _cmd_pipelines(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        if not chat:
+            return
+        if not self._pipelines_provider:
+            await context.bot.send_message(chat_id=chat.id, text="Pipelines service is offline.")
+            return
+        try:
+            pipelines = await self._pipelines_provider(5)
+        except Exception as e:
+            logger.warning("pipeline provider failed: %s", e)
+            await context.bot.send_message(chat_id=chat.id, text="Pipelines temporarily unavailable.")
+            return
+        if not pipelines:
+            await context.bot.send_message(
+                chat_id=chat.id,
+                text="No pipelines generated yet. Use /api/pipelines/generate.",
+            )
+            return
+        lines = ["<b>Recent Pipelines</b>"]
+        for i, p in enumerate(pipelines, start=1):
+            colab = p.get("colab_url") or ""
+            line = (
+                f"{i}. <b>{p.get('technique_name', '?')}</b> "
+                f"({p.get('task_type', '?')}, {p.get('status', '?')})"
+            )
+            if colab.startswith("https://"):
+                line += f"\n  <a href=\"{colab}\">Open in Colab</a>"
+            lines.append(line)
+        await context.bot.send_message(
+            chat_id=chat.id,
+            text="\n".join(lines),
+            parse_mode=ParseMode.HTML,
+            disable_web_page_preview=True,
+        )
+
+    async def _cmd_unsubscribe(self, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+        chat = update.effective_chat
+        if not chat:
+            return
+        self._delete_subscriber(chat.id)
+        await context.bot.send_message(
+            chat_id=chat.id,
+            text="You will no longer receive VectorMinds alerts. /start any time to subscribe again.",
+        )
+
+    # ── outbound delivery ─────────────────────────────────────
+
+    async def _send_to(self, chat_id: int | str, text: str) -> bool:
+        if not self.enabled or not self._app:
+            return False
+        try:
+            await self._app.bot.send_message(
+                chat_id=chat_id,
+                text=text,
+                parse_mode=ParseMode.HTML,
+                disable_web_page_preview=True,
+            )
+            self._sent_count += 1
+            return True
+        except TelegramError as e:
+            logger.warning("Telegram send to %s failed: %s", chat_id, e)
+            self._failed_count += 1
+            return False
+
+    async def broadcast(self, text: str) -> int:
+        """Send to every subscriber. Returns number of successful deliveries."""
+        if not self.enabled or not self._app:
+            return 0
+        chat_ids = self._list_subscriber_ids()
+        if not chat_ids:
+            logger.info("Telegram broadcast skipped (no subscribers).")
+            return 0
+        results = await asyncio.gather(
+            *(self._send_to(cid, text) for cid in chat_ids), return_exceptions=False
+        )
+        sent = sum(1 for ok in results if ok)
+        return sent
+
+    async def send_message(self, text: str, parse_mode: str = "HTML") -> bool:
+        """Backward-compatible single-call entrypoint that broadcasts to all subscribers."""
+        sent = await self.broadcast(text)
+        return sent > 0
+
+    async def send_trend_alert(self, technique: str, score: float, eta: int) -> bool:
+        msg = (
+            "<b>VectorMinds - New High-Impact Trend</b>\n\n"
+            f"<b>Technique:</b> {technique}\n"
+            f"<b>Emergence:</b> {score:.2f}\n"
+            f"<b>Mainstream ETA:</b> {eta} months\n\n"
+            "View in app or /trends"
+        )
+        return await self.send_message(msg)
+
+    async def send_pipeline_complete(
+        self, technique: str, task_type: str, metrics: dict, colab_url: str = ""
+    ) -> bool:
+        metrics_str = ", ".join(f"{k}: {v}" for k, v in (metrics or {}).items() if not isinstance(v, dict))
+        msg = (
+            "<b>VectorMinds - Training Pipeline Ready</b>\n\n"
+            f"<b>Technique:</b> {technique}\n"
+            f"<b>Task:</b> {task_type}\n"
+            f"<b>Highlights:</b> {metrics_str}"
+        )
+        if colab_url:
+            msg += f"\n\n<a href=\"{colab_url}\">Open in Colab</a>"
+        return await self.send_message(msg)
+
+    async def send_ingestion_summary(self, paper_count: int, repo_count: int) -> bool:
+        msg = (
+            "<b>VectorMinds - Ingestion Complete</b>\n\n"
+            f"<b>New Papers:</b> {paper_count}\n"
+            f"<b>New Repos:</b> {repo_count}\n"
+            f"<b>Total Signals:</b> {paper_count + repo_count}"
+        )
+        return await self.send_message(msg)
+
+    # ── subscriber persistence ────────────────────────────────
+
+    def _ensure_table(self) -> bool:
+        if not self.database:
+            return False
+        try:
+            self.database.ensure_telegram_subscribers_table()
+            return True
+        except Exception as e:
+            logger.warning("Telegram subscriber table not available: %s", e)
+            return False
+
+    def _upsert_subscriber(self, chat_id: int, username: Optional[str]) -> None:
+        if not self._ensure_table():
+            return
+        try:
+            self.database.upsert_telegram_subscriber(int(chat_id), username or "")
+            logger.info("Telegram subscriber added: %s", chat_id)
+        except Exception as e:
+            logger.warning("Failed to persist subscriber %s: %s", chat_id, e)
+
+    def _delete_subscriber(self, chat_id: int) -> None:
+        if not self._ensure_table():
+            return
+        try:
+            self.database.delete_telegram_subscriber(int(chat_id))
+            logger.info("Telegram subscriber removed: %s", chat_id)
+        except Exception as e:
+            logger.warning("Failed to delete subscriber %s: %s", chat_id, e)
+
+    def _list_subscriber_ids(self) -> list[int]:
+        if not self._ensure_table():
+            return []
+        try:
+            return list(self.database.list_telegram_subscriber_ids())
+        except Exception as e:
+            logger.warning("Failed to list subscribers: %s", e)
+            return []
+
+    def _subscriber_count(self) -> int:
+        return len(self._list_subscriber_ids())
+
+    # ── stats ────────────────────────────────────────────────
+
+    def get_stats(self) -> dict:
+        return {
+            "enabled": self.enabled,
+            "polling": self._polling_started,
+            "subscribers": self._subscriber_count(),
+            "messages_sent": self._sent_count,
+            "messages_failed": self._failed_count,
+            "now": datetime.now(timezone.utc).isoformat(),
+        }

embeddings/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Embeddings Package

embeddings/engine.py

@@ -0,0 +1,154 @@
+"""Vector Embedding Engine — Semantic core of VectorMinds.
+
+Handles hierarchical chunking, contrastive embeddings via BGE,
+semantic deduplication, and batch encoding.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+import config
+
+logger = logging.getLogger("vectorminds.embedding")
+
+
+class EmbeddingEngine:
+    """Manages text embedding using a sentence-transformer model."""
+
+    _instance: Optional["EmbeddingEngine"] = None
+    _model: Optional[SentenceTransformer] = None
+
+    @classmethod
+    def get_instance(cls) -> "EmbeddingEngine":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        self._model = None
+
+    def load_model(self):
+        """Load the embedding model (lazy initialization)."""
+        if self._model is None:
+            logger.info(f"Loading embedding model: {config.EMBEDDING_MODEL}")
+            self._model = SentenceTransformer(config.EMBEDDING_MODEL)
+            logger.info("Embedding model loaded successfully")
+
+    @property
+    def model(self) -> SentenceTransformer:
+        if self._model is None:
+            self.load_model()
+        return self._model
+
+    def embed_text(self, text: str) -> list[float]:
+        """Embed a single text string.
+
+        Args:
+            text: Input text to embed
+
+        Returns:
+            Embedding vector as list of floats
+        """
+        embedding = self.model.encode(text, normalize_embeddings=True)
+        return embedding.tolist()
+
+    def embed_batch(self, texts: list[str], batch_size: int = 32) -> list[list[float]]:
+        """Embed a batch of texts efficiently.
+
+        Args:
+            texts: List of input texts
+            batch_size: Encoding batch size
+
+        Returns:
+            List of embedding vectors
+        """
+        if not texts:
+            return []
+
+        logger.info(f"Embedding batch of {len(texts)} texts")
+        embeddings = self.model.encode(
+            texts,
+            batch_size=batch_size,
+            normalize_embeddings=True,
+            show_progress_bar=False,
+        )
+        return embeddings.tolist()
+
+    def chunk_text(self, text: str, max_chunk_size: int = 512) -> list[str]:
+        """Hierarchical chunking — split text into semantic chunks.
+
+        Implements paragraph-level chunking with overlap for better
+        retrieval granularity.
+
+        Args:
+            text: Input text to chunk
+            max_chunk_size: Maximum characters per chunk
+
+        Returns:
+            List of text chunks
+        """
+        if len(text) <= max_chunk_size:
+            return [text]
+
+        # Split by sentences first
+        sentences = text.replace(". ", ".\n").split("\n")
+        sentences = [s.strip() for s in sentences if s.strip()]
+
+        chunks = []
+        current_chunk = ""
+
+        for sentence in sentences:
+            if len(current_chunk) + len(sentence) + 1 <= max_chunk_size:
+                current_chunk += (" " if current_chunk else "") + sentence
+            else:
+                if current_chunk:
+                    chunks.append(current_chunk)
+                current_chunk = sentence
+
+        if current_chunk:
+            chunks.append(current_chunk)
+
+        return chunks if chunks else [text]
+
+    def compute_similarity(self, vec_a: list[float], vec_b: list[float]) -> float:
+        """Compute cosine similarity between two vectors."""
+        a = np.array(vec_a)
+        b = np.array(vec_b)
+        return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-8))
+
+    def is_duplicate(
+        self,
+        embedding: list[float],
+        existing_embeddings: list[list[float]],
+        threshold: float = None,
+    ) -> bool:
+        """Check if an embedding is a semantic duplicate of any existing embedding.
+
+        Args:
+            embedding: New embedding to check
+            existing_embeddings: List of existing embeddings
+            threshold: Similarity threshold (default from config)
+
+        Returns:
+            True if duplicate detected
+        """
+        if threshold is None:
+            threshold = config.DEDUP_SIMILARITY_THRESHOLD
+
+        if not existing_embeddings:
+            return False
+
+        new_vec = np.array(embedding)
+        for existing in existing_embeddings:
+            sim = float(
+                np.dot(new_vec, np.array(existing))
+                / (np.linalg.norm(new_vec) * np.linalg.norm(existing) + 1e-8)
+            )
+            if sim >= threshold:
+                return True
+        return False

embeddings/vector_store.py

@@ -0,0 +1,295 @@
+"""Qdrant Vector Store — Semantic search and storage layer.
+
+Wraps Qdrant client in in-memory mode for the hackathon MVP.
+Handles collection management, upsert, k-NN search, and novelty computation.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Optional
+
+import numpy as np
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance,
+    PointStruct,
+    VectorParams,
+    Filter,
+    FieldCondition,
+    MatchValue,
+)
+
+import config
+
+logger = logging.getLogger("vectormind.vectorstore")
+
+
+class VectorStore:
+    """Qdrant-based vector store for research signals."""
+
+    _instance: Optional["VectorStore"] = None
+
+    @classmethod
+    def get_instance(cls) -> "VectorStore":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        self.client: Optional[QdrantClient] = None
+        self.collection_name = config.QDRANT_COLLECTION
+        self.concept_graph: dict[str, list[dict]] = {} # Concept -> [neighboring concepts with time]
+
+    def initialize(self):
+        """Initialize Qdrant client and create collection."""
+        if config.QDRANT_HOST:
+            self.client = QdrantClient(
+                host=config.QDRANT_HOST, port=config.QDRANT_PORT
+            )
+        else:
+            # In-memory mode for hackathon
+            self.client = QdrantClient(":memory:")
+
+        # Create collection if it doesn't exist
+        collections = self.client.get_collections().collections
+        exists = any(c.name == self.collection_name for c in collections)
+
+        if not exists:
+            self.client.create_collection(
+                collection_name=self.collection_name,
+                vectors_config=VectorParams(
+                    size=config.EMBEDDING_DIM,
+                    distance=Distance.COSINE,
+                ),
+            )
+            logger.info(
+                f"Created Qdrant collection '{self.collection_name}' "
+                f"(dim={config.EMBEDDING_DIM})"
+            )
+
+    def upsert_signal(
+        self,
+        signal_id: str,
+        embedding: list[float],
+        payload: dict,
+    ):
+        """Store a research signal vector with metadata.
+
+        Args:
+            signal_id: Unique signal identifier
+            embedding: Vector embedding
+            payload: Metadata payload (title, source, scores, etc.)
+        """
+        self.client.upsert(
+            collection_name=self.collection_name,
+            points=[
+                PointStruct(
+                    id=signal_id.replace("-", "")[:32],  # Qdrant needs specific ID format
+                    vector=embedding,
+                    payload=payload,
+                )
+            ],
+        )
+
+    def upsert_batch(
+        self,
+        ids: list[str],
+        embeddings: list[list[float]],
+        payloads: list[dict],
+    ):
+        """Batch upsert multiple vectors."""
+        points = [
+            PointStruct(
+                id=idx,
+                vector=emb,
+                payload=pay,
+            )
+            for idx, (emb, pay) in enumerate(zip(embeddings, payloads))
+        ]
+        if points:
+            self.client.upsert(
+                collection_name=self.collection_name,
+                points=points,
+            )
+            logger.info(f"Upserted {len(points)} vectors")
+
+    def search(
+        self,
+        query_vector: list[float],
+        top_k: int = 10,
+        source_filter: Optional[str] = None,
+    ) -> list[dict]:
+        """Semantic similarity search.
+
+        Args:
+            query_vector: Query embedding
+            top_k: Number of results to return
+            source_filter: Optional filter by source type
+
+        Returns:
+            List of dicts with score and payload
+        """
+        query_filter = None
+        if source_filter:
+            query_filter = Filter(
+                must=[
+                    FieldCondition(
+                        key="source",
+                        match=MatchValue(value=source_filter),
+                    )
+                ]
+            )
+
+        response = self.client.query_points(
+            collection_name=self.collection_name,
+            query=query_vector,
+            limit=top_k,
+            query_filter=query_filter,
+            with_payload=True,
+        )
+        results = response.points
+
+        return [
+            {
+                "id": str(r.id),
+                "score": r.score,
+                "payload": r.payload,
+            }
+            for r in results
+        ]
+
+    def get_collection_count(self) -> int:
+        """Get total number of vectors in the collection."""
+        try:
+            info = self.client.get_collection(self.collection_name)
+            return info.points_count
+        except Exception:
+            return 0
+
+    def compute_novelty_score(
+        self,
+        embedding: list[float],
+        k: int = None,
+    ) -> float:
+        """Compute novelty score for a new embedding (Section 4.2 algorithm).
+
+        Steps:
+        1. Retrieve k nearest neighbors
+        2. Compute mean distance (d_mean) and min distance (d_min)
+        3. novelty = 0.6 * d_mean + 0.4 * d_min
+        4. Normalize to [0, 1]
+
+        Args:
+            embedding: New signal embedding
+            k: Number of neighbors (default from config)
+
+        Returns:
+            Novelty score in [0, 1]
+        """
+        if k is None:
+            k = config.NOVELTY_K_NEIGHBORS
+
+        count = self.get_collection_count()
+        if count == 0:
+            return 1.0  # First signal is maximally novel
+
+        # Adjust k if we have fewer points
+        actual_k = min(k, count)
+
+        response = self.client.query_points(
+            collection_name=self.collection_name,
+            query=embedding,
+            limit=actual_k,
+            with_payload=False,
+        )
+        results = response.points
+
+        if not results:
+            return 1.0
+
+        # Cosine distance = 1 - cosine_similarity. For unit-normalized embeddings
+        # this is in [0, 2] but in practice [0, ~0.6] dominates within a topic.
+        distances = [max(0.0, min(2.0, 1.0 - float(r.score))) for r in results]
+
+        d_mean = float(np.mean(distances))
+        d_min = float(np.min(distances))
+
+        # Weighted combination of mean+min distance (Section 4.2).
+        raw_novelty = (
+            config.NOVELTY_MEAN_WEIGHT * d_mean
+            + config.NOVELTY_MIN_WEIGHT * d_min
+        )
+
+        # Map raw distance into [0, 1] via a sigmoid centred at d=0.30. Below
+        # ~0.05 distance the novelty drops near 0; above ~0.55 it approaches 1
+        # but does not saturate the entire batch like the previous 5x linear
+        # scale did.
+        center = 0.30
+        slope = 8.0
+        z = slope * (raw_novelty - center)
+        novelty = 1.0 / (1.0 + 2.718281828 ** (-z))
+
+        return round(float(min(1.0, max(0.0, novelty))), 4)
+
+    def get_all_payloads(self, limit: int = 1000) -> list[dict]:
+        """Retrieve all stored payloads (for trend computation)."""
+        try:
+            results = self.client.scroll(
+                collection_name=self.collection_name,
+                limit=limit,
+                with_payload=True,
+                with_vectors=False,
+            )
+            return [
+                {"id": str(point.id), **point.payload}
+                for point in results[0]
+            ]
+        except Exception as e:
+            logger.error(f"Failed to scroll payloads: {e}")
+            return []
+
+    def get_vectors_for_projection(self, limit: int = 500) -> tuple[list, list]:
+        """Get vectors and payloads for 2D projection (t-SNE/UMAP viz).
+
+        Returns:
+            Tuple of (vectors, payloads)
+        """
+        try:
+            results = self.client.scroll(
+                collection_name=self.collection_name,
+                limit=limit,
+                with_payload=True,
+                with_vectors=True,
+            )
+            vectors = [point.vector for point in results[0]]
+            payloads = [point.payload for point in results[0]]
+            return vectors, payloads
+        except Exception as e:
+            logger.error(f"Failed to get vectors for projection: {e}")
+            return [], []
+    def build_temporal_graph(self) -> dict:
+        """Construct a graph of concepts tracking evolution over time.
+        
+        Follows Section 5.4: Temporal graph construction.
+        """
+        payloads = self.get_all_payloads()
+        if not payloads: return {}
+
+        # 1. Group by category/tag
+        graph = {}
+        for p in payloads:
+            tags = p.get("categories", [])
+            for tag in tags:
+                if tag not in graph: graph[tag] = []
+                graph[tag].append({
+                    "id": p.get("id"),
+                    "title": p.get("title"),
+                    "timestamp": p.get("timestamp"),
+                    "score": p.get("novelty_score", 0)
+                })
+        
+        self.concept_graph = graph
+        logger.info(f"Temporal graph built with {len(graph)} concept nodes")
+        return graph

ingestion/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Ingestion Package

ingestion/arxiv_crawler.py

@@ -0,0 +1,122 @@
+"""arXiv Crawler — Ingests research papers from arXiv API.
+
+Fetches recent papers by category, parses metadata, and returns
+normalized ResearchSignal objects.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Optional
+
+import arxiv
+import httpx
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.arxiv")
+
+
+class ArxivCrawler:
+    """Crawls arXiv for recent AI/ML papers."""
+
+    def __init__(self, categories: list[str], max_results: int = 50):
+        self.categories = categories
+        self.max_results = max_results
+        self.client = arxiv.Client()
+
+    async def fetch_recent_papers(
+        self,
+        query: Optional[str] = None,
+        category: Optional[str] = None,
+        max_results: Optional[int] = None,
+    ) -> list[ResearchSignal]:
+        """Fetch recent papers from arXiv.
+
+        Args:
+            query: Optional search query (e.g. 'transformer attention')
+            category: Specific arXiv category (e.g. 'cs.LG')
+            max_results: Override default max results
+
+        Returns:
+            List of ResearchSignal objects
+        """
+        n = max_results or self.max_results
+
+        # Build search query
+        if query:
+            search_query = query
+        elif category:
+            search_query = f"cat:{category}"
+        else:
+            # Search across all configured categories
+            cat_query = " OR ".join(f"cat:{c}" for c in self.categories)
+            search_query = cat_query
+
+        logger.info(f"Fetching arXiv papers: query='{search_query}', max={n}")
+
+        try:
+            search = arxiv.Search(
+                query=search_query,
+                max_results=n,
+                sort_by=arxiv.SortCriterion.SubmittedDate,
+                sort_order=arxiv.SortOrder.Descending,
+            )
+
+            signals = []
+            for result in self.client.results(search):
+                signal = ResearchSignal(
+                    source=SignalSource.ARXIV,
+                    source_id=result.entry_id,
+                    timestamp=result.published or datetime.utcnow(),
+                    title=result.title.strip().replace("\n", " "),
+                    raw_text=result.summary.strip().replace("\n", " "),
+                    authors=[a.name for a in result.authors],
+                    categories=[c for c in result.categories],
+                    url=result.entry_id,
+                    metadata={
+                        "pdf_url": result.pdf_url or "",
+                        "primary_category": result.primary_category,
+                        "comment": result.comment or "",
+                        "journal_ref": result.journal_ref or "",
+                        "doi": result.doi or "",
+                        "updated": (
+                            result.updated.isoformat() if result.updated else ""
+                        ),
+                    },
+                )
+                signals.append(signal)
+
+            logger.info(f"Fetched {len(signals)} papers from arXiv")
+            return signals
+
+        except Exception as e:
+            logger.error(f"arXiv fetch failed: {e}")
+            return []
+
+    async def fetch_by_ids(self, paper_ids: list[str]) -> list[ResearchSignal]:
+        """Fetch specific papers by their arXiv IDs."""
+        try:
+            search = arxiv.Search(id_list=paper_ids)
+            signals = []
+            for result in self.client.results(search):
+                signal = ResearchSignal(
+                    source=SignalSource.ARXIV,
+                    source_id=result.entry_id,
+                    timestamp=result.published or datetime.utcnow(),
+                    title=result.title.strip().replace("\n", " "),
+                    raw_text=result.summary.strip().replace("\n", " "),
+                    authors=[a.name for a in result.authors],
+                    categories=list(result.categories),
+                    url=result.entry_id,
+                    metadata={
+                        "pdf_url": result.pdf_url or "",
+                        "primary_category": result.primary_category,
+                    },
+                )
+                signals.append(signal)
+            return signals
+        except Exception as e:
+            logger.error(f"arXiv ID fetch failed: {e}")
+            return []

ingestion/blog_crawler.py

@@ -0,0 +1,72 @@
+"""Blog crawler using public RSS feeds from AI labs."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import httpx
+from bs4 import BeautifulSoup
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.blog")
+
+FEEDS = [
+    "https://openai.com/blog/rss.xml",
+    "https://deepmind.google/discover/blog/rss.xml",
+    "https://huggingface.co/blog/feed.xml",
+]
+
+
+class BlogCrawler:
+    async def fetch_blog_signals(self, max_results: int = 20) -> list[ResearchSignal]:
+        signals: list[ResearchSignal] = []
+        try:
+            async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+                for feed_url in FEEDS:
+                    resp = await client.get(feed_url)
+                    if resp.status_code != 200:
+                        continue
+                    soup = BeautifulSoup(resp.text, "xml")
+                    for item in soup.find_all(["item", "entry"]):
+                        title = (item.find("title").text if item.find("title") else "").strip()
+                        summary_node = item.find("description") or item.find("summary")
+                        summary = (
+                            summary_node.text.strip() if summary_node and summary_node.text else ""
+                        )
+                        link_node = item.find("link")
+                        url = ""
+                        if link_node:
+                            url = link_node.get("href", "") or link_node.text or ""
+                        date_node = item.find("pubDate") or item.find("updated")
+                        ts = datetime.utcnow()
+                        if date_node and date_node.text:
+                            try:
+                                ts = datetime.strptime(
+                                    date_node.text[:25], "%a, %d %b %Y %H:%M:%S"
+                                )
+                            except Exception:
+                                ts = datetime.utcnow()
+
+                        if not title:
+                            continue
+                        signals.append(
+                            ResearchSignal(
+                                source=SignalSource.BLOG,
+                                source_id=f"blog:{abs(hash(url or title))}",
+                                timestamp=ts,
+                                title=title,
+                                raw_text=summary or title,
+                                authors=["AI Blog"],
+                                categories=["blog", "ai"],
+                                url=url,
+                                metadata={"feed_url": feed_url, "source_system": "rss"},
+                            )
+                        )
+                        if len(signals) >= max_results:
+                            return signals
+            return signals
+        except Exception as e:
+            logger.error(f"Blog fetch failed: {e}")
+            return []

ingestion/github_crawler.py

@@ -0,0 +1,161 @@
+"""GitHub Crawler — Discovers trending ML/AI repositories.
+
+Uses the GitHub REST API (unauthenticated / free) to find repositories
+with high recent activity in machine learning topics.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta
+from typing import Optional
+
+import httpx
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.github")
+
+GITHUB_API = "https://api.github.com"
+
+
+class GitHubCrawler:
+    """Crawls GitHub for trending ML/AI repositories."""
+
+    def __init__(
+        self,
+        languages: list[str] | None = None,
+        max_results: int = 30,
+        token: str = "",
+    ):
+        self.languages = languages or ["python"]
+        self.max_results = max_results
+        headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "VectorMinds-Research-Intelligence",
+        }
+        if token:
+            headers["Authorization"] = f"token {token}"
+        self.headers = headers
+
+    async def fetch_trending_repos(
+        self,
+        topic: str = "machine-learning",
+        days_back: int = 7,
+        max_results: Optional[int] = None,
+    ) -> list[ResearchSignal]:
+        """Fetch trending ML repos from GitHub.
+
+        Args:
+            topic: GitHub topic to search (e.g. 'machine-learning', 'deep-learning')
+            days_back: Look back window in days
+            max_results: Override default max results
+
+        Returns:
+            List of ResearchSignal objects
+        """
+        n = max_results or self.max_results
+        since_date = (datetime.utcnow() - timedelta(days=days_back)).strftime(
+            "%Y-%m-%d"
+        )
+
+        # Build search query for trending ML repos
+        lang_query = " ".join(f"language:{l}" for l in self.languages)
+        query = f"topic:{topic} {lang_query} created:>{since_date} stars:>5"
+
+        logger.info(f"Fetching GitHub repos: query='{query}', max={n}")
+
+        try:
+            async with httpx.AsyncClient(timeout=30.0) as client:
+                resp = await client.get(
+                    f"{GITHUB_API}/search/repositories",
+                    params={
+                        "q": query,
+                        "sort": "stars",
+                        "order": "desc",
+                        "per_page": min(n, 100),
+                    },
+                    headers=self.headers,
+                )
+                resp.raise_for_status()
+                data = resp.json()
+
+            signals = []
+            for repo in data.get("items", [])[:n]:
+                # Compute stars-per-day acceleration
+                created = datetime.strptime(
+                    repo["created_at"], "%Y-%m-%dT%H:%M:%SZ"
+                )
+                age_days = max((datetime.utcnow() - created).days, 1)
+                stars_per_day = repo.get("stargazers_count", 0) / age_days
+
+                signal = ResearchSignal(
+                    source=SignalSource.GITHUB,
+                    source_id=repo["full_name"],
+                    timestamp=datetime.strptime(
+                        repo["updated_at"], "%Y-%m-%dT%H:%M:%SZ"
+                    ),
+                    title=repo["full_name"],
+                    raw_text=repo.get("description", "") or "",
+                    authors=[repo["owner"]["login"]],
+                    categories=repo.get("topics", []) or [],
+                    url=repo["html_url"],
+                    metadata={
+                        "stars": repo.get("stargazers_count", 0),
+                        "forks": repo.get("forks_count", 0),
+                        "watchers": repo.get("watchers_count", 0),
+                        "language": repo.get("language", ""),
+                        "open_issues": repo.get("open_issues_count", 0),
+                        "stars_per_day": round(stars_per_day, 2),
+                        "license": (
+                            repo.get("license", {}) or {}
+                        ).get("spdx_id", ""),
+                        "size_kb": repo.get("size", 0),
+                        "created_at": repo["created_at"],
+                    },
+                )
+                signals.append(signal)
+
+            logger.info(f"Fetched {len(signals)} repos from GitHub")
+            return signals
+
+        except Exception as e:
+            logger.error(f"GitHub fetch failed: {e}")
+            return []
+
+    async def fetch_repo_details(self, full_name: str) -> Optional[ResearchSignal]:
+        """Fetch details for a specific repository."""
+        try:
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.get(
+                    f"{GITHUB_API}/repos/{full_name}",
+                    headers=self.headers,
+                )
+                resp.raise_for_status()
+                repo = resp.json()
+
+            created = datetime.strptime(repo["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+            age_days = max((datetime.utcnow() - created).days, 1)
+            stars_per_day = repo.get("stargazers_count", 0) / age_days
+
+            return ResearchSignal(
+                source=SignalSource.GITHUB,
+                source_id=repo["full_name"],
+                timestamp=datetime.strptime(
+                    repo["updated_at"], "%Y-%m-%dT%H:%M:%SZ"
+                ),
+                title=repo["full_name"],
+                raw_text=repo.get("description", "") or "",
+                authors=[repo["owner"]["login"]],
+                categories=repo.get("topics", []) or [],
+                url=repo["html_url"],
+                metadata={
+                    "stars": repo.get("stargazers_count", 0),
+                    "forks": repo.get("forks_count", 0),
+                    "stars_per_day": round(stars_per_day, 2),
+                    "language": repo.get("language", ""),
+                },
+            )
+        except Exception as e:
+            logger.error(f"GitHub repo fetch failed: {e}")
+            return None

ingestion/patent_crawler.py

@@ -0,0 +1,94 @@
+"""Patent crawler using PatentsView public API."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import httpx
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.patents")
+
+PATENTSVIEW_URL = "https://api.patentsview.org/patents/query"
+
+
+class PatentCrawler:
+    """Fetches recent AI-related patents from public PatentsView API."""
+
+    async def fetch_recent_patents(self, max_results: int = 20) -> list[ResearchSignal]:
+        query = {
+            "_or": [
+                {"_text_any": {"patent_title": "artificial intelligence"}},
+                {"_text_any": {"patent_title": "machine learning"}},
+                {"_text_any": {"patent_abstract": "neural network"}},
+                {"_text_any": {"patent_abstract": "transformer model"}},
+            ]
+        }
+        fields = [
+            "patent_number",
+            "patent_title",
+            "patent_date",
+            "patent_abstract",
+            "patent_type",
+            "assignee_organization",
+        ]
+
+        try:
+            async with httpx.AsyncClient(timeout=25.0) as client:
+                resp = await client.post(
+                    PATENTSVIEW_URL,
+                    json={
+                        "q": query,
+                        "f": fields,
+                        "o": {"per_page": max_results},
+                    },
+                )
+                resp.raise_for_status()
+                data = resp.json()
+        except Exception as e:
+            logger.error(f"PatentsView fetch failed: {e}")
+            return []
+
+        patents = data.get("patents", [])
+        signals: list[ResearchSignal] = []
+        for p in patents:
+            patent_number = p.get("patent_number", "")
+            title = p.get("patent_title", "") or ""
+            abstract = p.get("patent_abstract", "") or ""
+            assignees = p.get("assignees", []) or []
+            orgs = [
+                a.get("assignee_organization", "")
+                for a in assignees
+                if isinstance(a, dict)
+            ]
+            patent_date = p.get("patent_date", "")
+            try:
+                ts = datetime.strptime(patent_date, "%Y-%m-%d")
+            except Exception:
+                ts = datetime.utcnow()
+
+            signals.append(
+                ResearchSignal(
+                    source=SignalSource.PATENT,
+                    source_id=patent_number or title[:64],
+                    timestamp=ts,
+                    title=title or f"Patent {patent_number}",
+                    raw_text=abstract[:4000],
+                    authors=orgs[:5],
+                    categories=["patent", "ai"],
+                    url=(
+                        f"https://patents.google.com/patent/{patent_number}"
+                        if patent_number
+                        else ""
+                    ),
+                    metadata={
+                        "patent_number": patent_number,
+                        "patent_type": p.get("patent_type", ""),
+                        "assignees": orgs[:10],
+                        "source_system": "patentsview",
+                    },
+                )
+            )
+        return signals

ingestion/pdf_parser.py

@@ -0,0 +1,83 @@
+"""PDF Parser — Section-Aware Research Document Processing.
+
+Extracts structured text from research PDFs and performs hierarchical
+chunking for granular vector indexing.
+"""
+
+import re
+import logging
+from typing import Dict, List
+
+logger = logging.getLogger("vectormind.ingestion")
+
+class PDFParser:
+    """Parses research PDFs into structured sections and chunks."""
+
+    def __init__(self):
+        # Section header patterns
+        self.section_patterns = {
+            "abstract": re.compile(r"^abstract", re.IGNORECASE),
+            "introduction": re.compile(r"^1\.?\s+introduction|^introduction", re.IGNORECASE),
+            "methodology": re.compile(r"^2\.?\s+methods|^methods|^methodology", re.IGNORECASE),
+            "results": re.compile(r"^3\.?\s+results|^results", re.IGNORECASE),
+            "discussion": re.compile(r"^4\.?\s+discussion|^discussion", re.IGNORECASE),
+            "conclusion": re.compile(r"^5\.?\s+conclusion|^conclusion", re.IGNORECASE),
+            "references": re.compile(r"^references", re.IGNORECASE),
+        }
+
+    def parse_text(self, text: str) -> Dict[str, str]:
+        """Parse raw text into sections based on headers."""
+        sections = {"abstract": "", "main_text": "", "full_text": text}
+        lines = text.split('\n')
+        
+        current_section = "abstract"
+        
+        for line in lines:
+            line_clean = line.strip()
+            if not line_clean:
+                continue
+                
+            # Check for section transition
+            found_header = False
+            for section_name, pattern in self.section_patterns.items():
+                if pattern.match(line_clean):
+                    current_section = section_name
+                    found_header = True
+                    break
+            
+            if not found_header:
+                if current_section in sections:
+                    sections[current_section] += line + "\n"
+                else:
+                    sections[current_section] = line + "\n"
+                    
+        return sections
+
+    def hierarchical_chunking(self, sections: Dict[str, str], chunk_size: int = 1000) -> List[Dict]:
+        """Create chunks at different granularity levels."""
+        chunks = []
+        
+        # 1. Abstract level (high-level)
+        if sections.get("abstract"):
+            chunks.append({
+                "level": "abstract",
+                "text": sections["abstract"],
+                "metadata": {"type": "summary"}
+            })
+            
+        # 2. Section level (paragraph-aware)
+        for name, content in sections.items():
+            if name in ["full_text", "references"]: continue
+            
+            # Simple paragraph splitting
+            paragraphs = content.split('\n\n')
+            for i, p in enumerate(paragraphs):
+                if len(p.strip()) < 50: continue
+                chunks.append({
+                    "level": "section",
+                    "section": name,
+                    "text": p.strip(),
+                    "metadata": {"para_idx": i}
+                })
+                
+        return chunks

ingestion/schema.py

@@ -0,0 +1,142 @@
+"""Research Signal canonical schema.
+
+All ingested data is normalized into this schema regardless of source.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class SignalSource(str, Enum):
+    ARXIV = "arxiv"
+    GITHUB = "github"
+    PATENT = "patent"
+    STARTUP = "startup"
+    SOCIAL = "social"
+    BLOG = "blog"
+
+
+class ResearchSignal(BaseModel):
+    """Canonical Research Signal — the universal data unit in VectorMinds."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    source: SignalSource
+    source_id: str = ""  # e.g. arXiv paper ID, GitHub repo full_name
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    title: str
+    raw_text: str  # abstract or description
+    authors: list[str] = Field(default_factory=list)
+    categories: list[str] = Field(default_factory=list)
+    url: str = ""
+    embedding: list[float] = Field(default_factory=list)
+    novelty_score: float = 0.0
+    impact_score: float = 0.0
+    metadata: dict = Field(default_factory=dict)
+
+    # Computed fields (populated by Reasoning Agent)
+    technique_name: str = ""
+    technical_brief: str = ""
+    cross_source_signals: dict = Field(default_factory=dict)
+
+
+class TrendEntry(BaseModel):
+    """A ranked entry in the Trend Leaderboard."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    rank: int = 0
+    technique_name: str
+    description: str = ""
+    emergence_score: float = 0.0
+    novelty_score: float = 0.0
+    impact_score: float = 0.0
+    mainstream_eta_months: int = 12
+    confidence: float = 0.0
+    source_signals: dict = Field(default_factory=dict)
+    competitive_landscape: list[str] = Field(default_factory=list)
+    risk_factors: list[str] = Field(default_factory=list)
+    related_techniques: list[str] = Field(default_factory=list)
+    paper_count: int = 0
+    github_stars: int = 0
+    first_seen: datetime = Field(default_factory=datetime.utcnow)
+    last_updated: datetime = Field(default_factory=datetime.utcnow)
+    signal_ids: list[str] = Field(default_factory=list)
+
+
+class ProductBlueprint(BaseModel):
+    """A complete product blueprint generated from a high-scoring technique."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    technique_name: str
+    trend_id: str = ""
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+    problem_statement: str = ""
+    market_size: str = ""
+    technical_implementation: str = ""
+    architecture_decisions: list[str] = Field(default_factory=list)
+    differentiation_strategy: str = ""
+    dataset_requirements: str = ""
+    go_to_market: str = ""
+    risk_assessment: str = ""
+    first_90_day_milestones: list[str] = Field(default_factory=list)
+    suggested_stack: list[str] = Field(default_factory=list)
+
+
+class MLPipeline(BaseModel):
+    """A generated ML training pipeline."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    technique_name: str
+    blueprint_id: str = ""
+    task_type: str = ""  # one of SUPPORTED_TASK_CATEGORIES
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+    dataset_name: str = ""
+    dataset_source: str = ""
+    model_architecture: str = ""
+    notebook_content: str = ""
+    colab_url: str = ""
+    status: str = "generated"  # generated, training, completed, failed
+    metrics: dict = Field(default_factory=dict)
+    model_card: str = ""
+
+
+class AgentEvent(BaseModel):
+    """Event message passed between agents via the message bus."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    topic: str
+    source_agent: str
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    payload: dict = Field(default_factory=dict)
+
+
+class UserFeedback(BaseModel):
+    """User feedback on a prediction or blueprint."""
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    target_id: str  # trend_id or blueprint_id
+    target_type: str  # "trend" or "blueprint"
+    action: str  # "upvote" or "downvote"
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+
+
+class PlatformStats(BaseModel):
+    """Live platform statistics for the dashboard."""
+
+    total_papers: int = 0
+    total_github_repos: int = 0
+    total_signals: int = 0
+    active_trends: int = 0
+    blueprints_generated: int = 0
+    pipelines_launched: int = 0
+    avg_novelty_score: float = 0.0
+    novelty_distribution: list[float] = Field(default_factory=list)
+    agents_status: dict = Field(default_factory=dict)
+    last_ingestion: Optional[datetime] = None

ingestion/social_crawler.py

@@ -0,0 +1,66 @@
+"""Social signal crawler (Hacker News public API)."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import httpx
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.social")
+
+HN_TOP = "https://hacker-news.firebaseio.com/v0/topstories.json"
+HN_ITEM = "https://hacker-news.firebaseio.com/v0/item/{item_id}.json"
+
+
+class SocialCrawler:
+    async def fetch_hn_signals(self, max_results: int = 30) -> list[ResearchSignal]:
+        try:
+            async with httpx.AsyncClient(timeout=25.0) as client:
+                ids_resp = await client.get(HN_TOP)
+                ids_resp.raise_for_status()
+                top_ids = (ids_resp.json() or [])[: max_results * 2]
+
+                signals: list[ResearchSignal] = []
+                for item_id in top_ids:
+                    item_resp = await client.get(HN_ITEM.format(item_id=item_id))
+                    if item_resp.status_code != 200:
+                        continue
+                    item = item_resp.json() or {}
+                    if item.get("type") != "story":
+                        continue
+                    title = (item.get("title") or "").lower()
+                    text = (item.get("text") or "").lower()
+                    if not any(
+                        k in (title + " " + text)
+                        for k in ("ai", "llm", "machine learning", "transformer", "agent")
+                    ):
+                        continue
+
+                    ts = datetime.utcfromtimestamp(item.get("time", 0) or 0)
+                    signals.append(
+                        ResearchSignal(
+                            source=SignalSource.SOCIAL,
+                            source_id=f"HN-{item_id}",
+                            timestamp=ts if ts.year > 2000 else datetime.utcnow(),
+                            title=item.get("title", "HN story"),
+                            raw_text=item.get("text", "") or item.get("title", ""),
+                            authors=[item.get("by", "hn_user")],
+                            categories=["hacker-news", "social"],
+                            url=item.get("url", f"https://news.ycombinator.com/item?id={item_id}"),
+                            metadata={
+                                "hn_id": item_id,
+                                "score": item.get("score", 0),
+                                "descendants": item.get("descendants", 0),
+                                "source_system": "hackernews",
+                            },
+                        )
+                    )
+                    if len(signals) >= max_results:
+                        break
+                return signals
+        except Exception as e:
+            logger.error(f"HN fetch failed: {e}")
+            return []

ingestion/startup_crawler.py

@@ -0,0 +1,69 @@
+"""Startup signal crawler using public startup/news RSS feeds."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import httpx
+from bs4 import BeautifulSoup
+
+from ingestion.schema import ResearchSignal, SignalSource
+
+logger = logging.getLogger("vectorminds.startups")
+
+STARTUP_FEEDS = [
+    "https://techcrunch.com/category/startups/feed/",
+    "https://www.ycombinator.com/blog/rss/",
+]
+
+
+class StartupCrawler:
+    async def fetch_startup_signals(self, max_results: int = 20) -> list[ResearchSignal]:
+        signals: list[ResearchSignal] = []
+        try:
+            async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+                for feed_url in STARTUP_FEEDS:
+                    resp = await client.get(feed_url)
+                    if resp.status_code != 200:
+                        continue
+                    soup = BeautifulSoup(resp.text, "xml")
+                    for item in soup.find_all(["item", "entry"]):
+                        title = (item.find("title").text if item.find("title") else "").strip()
+                        if not title:
+                            continue
+                        text = (
+                            (item.find("description").text if item.find("description") else "")
+                            .strip()
+                            .lower()
+                        )
+                        if not any(
+                            k in (title.lower() + " " + text)
+                            for k in ("ai", "machine learning", "llm", "model")
+                        ):
+                            continue
+
+                        link_node = item.find("link")
+                        url = ""
+                        if link_node:
+                            url = link_node.get("href", "") or link_node.text or ""
+
+                        signals.append(
+                            ResearchSignal(
+                                source=SignalSource.STARTUP,
+                                source_id=f"startup:{abs(hash(url or title))}",
+                                timestamp=datetime.utcnow(),
+                                title=title,
+                                raw_text=text or title,
+                                authors=["startup-news"],
+                                categories=["startup", "funding"],
+                                url=url,
+                                metadata={"feed_url": feed_url, "source_system": "rss"},
+                            )
+                        )
+                        if len(signals) >= max_results:
+                            return signals
+            return signals
+        except Exception as e:
+            logger.error(f"Startup feed fetch failed: {e}")
+            return []

intelligence/__init__.py

@@ -0,0 +1 @@
+# VectorMinds Intelligence Package

intelligence/blueprint_engine.py

@@ -0,0 +1,358 @@
+"""Product Blueprint Engine — Generates startup-ready product briefs.
+
+Takes a high-scoring technique and generates a complete product blueprint
+using LLM (Gemini) or mock data for demo.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from datetime import datetime
+from typing import Optional
+
+import httpx
+
+from ingestion.schema import ProductBlueprint, TrendEntry
+import config
+
+logger = logging.getLogger("vectorminds.blueprint")
+
+
+def _coerce_text(value, indent: int = 0) -> str:
+    """Coerce a Gemini value (string|list|dict|number|None) into a readable string.
+
+    Lists become bullet lines; dicts become "Key:\\n  - item" sections. Used because
+    the LLM sometimes returns nested objects for fields the schema types as ``str``.
+    """
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    pad = "  " * indent
+    if isinstance(value, list):
+        return "\n".join(f"{pad}- {_coerce_text(v, indent + 1).lstrip()}" for v in value)
+    if isinstance(value, dict):
+        chunks = []
+        for k, v in value.items():
+            label = str(k).replace("_", " ").strip()
+            inner = _coerce_text(v, indent + 1)
+            if "\n" in inner or (isinstance(v, (list, dict))):
+                chunks.append(f"{pad}{label}:\n{inner}")
+            else:
+                chunks.append(f"{pad}{label}: {inner}")
+        return "\n".join(chunks)
+    return str(value)
+
+
+def _coerce_str_list(value) -> list[str]:
+    """Force a value into a list[str], coercing nested dicts/lists to readable lines."""
+    if value is None:
+        return []
+    if isinstance(value, list):
+        return [_coerce_text(v).strip() for v in value if v is not None]
+    if isinstance(value, dict):
+        return [f"{k}: {_coerce_text(v).strip()}" for k, v in value.items()]
+    return [str(value)]
+
+
+def _repair_truncated_json(text: str) -> Optional[dict]:
+    """Best-effort recovery of a truncated JSON object from a Gemini response.
+
+    Closes any open string, then closes any unbalanced ``[`` / ``{`` brackets,
+    in stack order. Returns ``None`` if the result still does not parse.
+    """
+    if not text:
+        return None
+    s = text.strip()
+    if not s.startswith("{"):
+        start = s.find("{")
+        if start == -1:
+            return None
+        s = s[start:]
+
+    in_str = False
+    escape = False
+    stack: list[str] = []
+    for ch in s:
+        if escape:
+            escape = False
+            continue
+        if ch == "\\":
+            escape = True
+            continue
+        if ch == '"':
+            in_str = not in_str
+            continue
+        if in_str:
+            continue
+        if ch in "{[":
+            stack.append("}" if ch == "{" else "]")
+        elif ch in "}]" and stack and stack[-1] == ch:
+            stack.pop()
+
+    repaired = s
+    if in_str:
+        repaired += '"'
+    while stack:
+        repaired += stack.pop()
+
+    repaired = repaired.rstrip(",")
+    try:
+        return json.loads(repaired)
+    except Exception:
+        return None
+
+# ─── Mock Blueprints for Demo ─────────────────────────────────
+MOCK_BLUEPRINTS = {
+    "default": {
+        "problem_statement": (
+            "Enterprise organizations process millions of documents daily — contracts, "
+            "reports, emails, compliance filings — but existing NLP solutions are limited "
+            "by transformer context windows (typically 4K-8K tokens). Documents exceeding "
+            "this limit require chunking strategies that lose cross-section context, "
+            "resulting in 15-30% accuracy degradation on long-document tasks. The market "
+            "for intelligent document processing is $4.2B (2024) growing at 28% CAGR."
+        ),
+        "market_size": "$4.2B (2024), projected $12.1B by 2028 at 28% CAGR",
+        "technical_implementation": (
+            "Build a document intelligence API powered by the selected technique. "
+            "Architecture: (1) Document ingestion service with OCR and layout detection, "
+            "(2) Adaptive chunking engine that preserves cross-reference context, "
+            "(3) Core inference engine using the technique for unlimited-context processing, "
+            "(4) Structured output layer with JSON/XML schema enforcement, "
+            "(5) REST API with streaming support for real-time processing. "
+            "Deploy on AWS with auto-scaling GPU instances (A10G for inference)."
+        ),
+        "architecture_decisions": [
+            "Use streaming inference to handle arbitrarily long documents",
+            "Implement a hybrid retrieval + full-context approach for optimal accuracy",
+            "Deploy as a containerized microservice for horizontal scaling",
+            "Cache embeddings in Redis for repeated document access patterns",
+        ],
+        "differentiation_strategy": (
+            "Unlike existing solutions (AWS Textract, Google Document AI, Azure Form "
+            "Recognizer), this product handles documents of ANY length without chunking "
+            "degradation. The core moat is the technique's linear-time complexity, "
+            "enabling 100x longer context at 1/10th the cost. Additional moats: "
+            "proprietary fine-tuning on 500K enterprise documents, and a self-improving "
+            "feedback loop where corrections from users improve the model continuously."
+        ),
+        "dataset_requirements": (
+            "Initial training: (1) DocVQA (50K document-question pairs), "
+            "(2) SQuAD 2.0 for reading comprehension baseline, "
+            "(3) Contract Understanding Atticus Dataset (CUAD) for legal domain, "
+            "(4) FUNSD for form understanding. Proprietary data collection: "
+            "Partner with 3 enterprise customers for anonymized document datasets. "
+            "Synthetic data: Generate 100K long-document QA pairs using GPT-4."
+        ),
+        "go_to_market": (
+            "Target: Legal tech firms and compliance teams (high document volume, "
+            "high accuracy requirements). Channel: Direct sales to Top-50 law firms, "
+            "integration partnerships with existing DMS providers (NetDocuments, iManage). "
+            "Pricing: Usage-based API pricing — $0.01/page for standard, $0.05/page for "
+            "premium with human-in-the-loop verification. First 90 days: 3 design partners, "
+            "10K documents processed, 95%+ accuracy on standard benchmarks."
+        ),
+        "risk_assessment": (
+            "Technical risks: (1) Technique may not generalize to all document types — "
+            "mitigate with domain-specific fine-tuning. (2) Inference latency on very "
+            "long documents (>100 pages) — mitigate with streaming and caching. "
+            "Market risks: (1) Incumbent response — AWS/Google/Azure may adopt similar "
+            "techniques within 12 months — speed to market is critical. "
+            "Competitive risks: (1) Several well-funded startups in adjacent space — "
+            "differentiate on long-context capability."
+        ),
+        "first_90_day_milestones": [
+            "Week 1-2: Core model integration and API scaffold",
+            "Week 3-4: Document ingestion pipeline with OCR",
+            "Week 5-6: Fine-tune on DocVQA and CUAD datasets",
+            "Week 7-8: API deployment with auth and rate limiting",
+            "Week 9-10: First design partner onboarding",
+            "Week 11-12: Benchmark publication and Product Hunt launch",
+        ],
+        "suggested_stack": [
+            "PyTorch + HuggingFace Transformers",
+            "FastAPI + Pydantic",
+            "Redis for caching",
+            "PostgreSQL for metadata",
+            "AWS ECS + A10G GPU instances",
+            "Stripe for billing",
+        ],
+    }
+}
+
+
+class BlueprintEngine:
+    """Generates product blueprints from high-scoring research techniques."""
+
+    def __init__(self):
+        self.generated_blueprints: dict[str, ProductBlueprint] = {}
+
+    async def generate_blueprint(
+        self,
+        trend: TrendEntry,
+        additional_context: str = "",
+    ) -> ProductBlueprint:
+        """Generate a complete product blueprint for a technique.
+
+        Args:
+            trend: The trend entry to generate a blueprint for
+            additional_context: Additional research context
+
+        Returns:
+            Complete ProductBlueprint
+        """
+        logger.info(f"Generating blueprint for: {trend.technique_name}")
+
+        if config.USE_MOCK_LLM or not config.LLM_API_KEY:
+            blueprint = self._generate_mock_blueprint(trend)
+        else:
+            blueprint = await self._generate_llm_blueprint(trend, additional_context)
+
+        self.generated_blueprints[blueprint.id] = blueprint
+        logger.info(f"Blueprint generated: {blueprint.id}")
+        return blueprint
+
+    async def _generate_llm_blueprint(
+        self, trend: TrendEntry, context: str
+    ) -> ProductBlueprint:
+        """Generate blueprint using Gemini LLM."""
+        prompt = (
+            f"Generate a complete startup product blueprint based on this "
+            f"emerging AI technique:\n\n"
+            f"Technique: {trend.technique_name}\n"
+            f"Description: {trend.description}\n"
+            f"Emergence Score: {trend.emergence_score}\n"
+            f"Impact Score: {trend.impact_score}\n"
+            f"Mainstream ETA: {trend.mainstream_eta_months} months\n\n"
+            f"Additional context:\n{context[:1500]}\n\n"
+            f"Please provide a JSON response with these fields:\n"
+            f"problem_statement, market_size, technical_implementation, "
+            f"architecture_decisions (list), differentiation_strategy, "
+            f"dataset_requirements, go_to_market, risk_assessment, "
+            f"first_90_day_milestones (list), suggested_stack (list)"
+        )
+
+        try:
+            async with httpx.AsyncClient(timeout=60.0) as client:
+                full_prompt = (
+                    "You are a senior product strategist and AI architect. "
+                    "Generate detailed, actionable product blueprints.\n"
+                    "Return ONLY valid JSON with these keys: "
+                    "problem_statement, market_size, technical_implementation, "
+                    "architecture_decisions, differentiation_strategy, dataset_requirements, "
+                    "go_to_market, risk_assessment, first_90_day_milestones, suggested_stack.\n\n"
+                    f"{prompt}"
+                )
+                payload = {
+                    "contents": [{"parts": [{"text": full_prompt}]}],
+                    "generationConfig": {
+                        "temperature": 0.7,
+                        "maxOutputTokens": 8192,
+                        "responseMimeType": "application/json",
+                    },
+                }
+
+                # Gemini occasionally returns 503/overloaded. Retry with simple
+                # exponential backoff (3 attempts, 1.5s/3s/6s) before falling
+                # back to the mock template.
+                resp = None
+                last_exc: Optional[Exception] = None
+                for attempt, delay in enumerate((1.5, 3.0, 6.0), start=1):
+                    try:
+                        resp = await client.post(
+                            f"{config.GEMINI_BASE_URL}/models/{config.LLM_MODEL}:generateContent",
+                            params={"key": config.LLM_API_KEY},
+                            headers={"Content-Type": "application/json"},
+                            json=payload,
+                        )
+                        if resp.status_code in (500, 502, 503, 504, 429):
+                            logger.warning(
+                                "Gemini blueprint attempt %s returned %s; retrying",
+                                attempt,
+                                resp.status_code,
+                            )
+                            last_exc = httpx.HTTPStatusError(
+                                f"Gemini transient {resp.status_code}",
+                                request=resp.request,
+                                response=resp,
+                            )
+                            await asyncio.sleep(delay)
+                            continue
+                        last_exc = None
+                        break
+                    except (httpx.ConnectError, httpx.ReadTimeout, httpx.RemoteProtocolError) as e:
+                        last_exc = e
+                        logger.warning("Gemini blueprint network error attempt %s: %s", attempt, e)
+                        await asyncio.sleep(delay)
+                        continue
+                if resp is None or last_exc is not None:
+                    if last_exc is not None:
+                        raise last_exc
+                    raise RuntimeError("Gemini blueprint: no response")
+                resp.raise_for_status()
+                data = resp.json()
+                candidates = data.get("candidates", [])
+                if not candidates:
+                    raise ValueError("No Gemini candidates returned")
+                finish_reason = candidates[0].get("finishReason", "")
+                parts = candidates[0].get("content", {}).get("parts", [])
+                content = "".join(p.get("text", "") for p in parts if isinstance(p, dict)).strip()
+                if not content:
+                    raise ValueError("Empty Gemini response text")
+                try:
+                    bp_data = json.loads(content)
+                except json.JSONDecodeError as je:
+                    if finish_reason == "MAX_TOKENS":
+                        logger.warning(
+                            "Gemini blueprint truncated by MAX_TOKENS; attempting repair"
+                        )
+                    bp_data = _repair_truncated_json(content)
+                    if bp_data is None:
+                        raise je
+
+                return ProductBlueprint(
+                    technique_name=trend.technique_name,
+                    trend_id=trend.id,
+                    problem_statement=_coerce_text(bp_data.get("problem_statement", "")),
+                    market_size=_coerce_text(bp_data.get("market_size", "")),
+                    technical_implementation=_coerce_text(bp_data.get("technical_implementation", "")),
+                    architecture_decisions=_coerce_str_list(bp_data.get("architecture_decisions", [])),
+                    differentiation_strategy=_coerce_text(bp_data.get("differentiation_strategy", "")),
+                    dataset_requirements=_coerce_text(bp_data.get("dataset_requirements", "")),
+                    go_to_market=_coerce_text(bp_data.get("go_to_market", "")),
+                    risk_assessment=_coerce_text(bp_data.get("risk_assessment", "")),
+                    first_90_day_milestones=_coerce_str_list(bp_data.get("first_90_day_milestones", [])),
+                    suggested_stack=_coerce_str_list(bp_data.get("suggested_stack", [])),
+                )
+        except Exception as e:
+            logger.error(f"LLM blueprint generation failed: {e}")
+            return self._generate_mock_blueprint(trend)
+
+    def _generate_mock_blueprint(self, trend: TrendEntry) -> ProductBlueprint:
+        """Generate a mock blueprint for demo purposes."""
+        mock = MOCK_BLUEPRINTS["default"]
+        return ProductBlueprint(
+            technique_name=trend.technique_name,
+            trend_id=trend.id,
+            problem_statement=mock["problem_statement"],
+            market_size=mock["market_size"],
+            technical_implementation=mock["technical_implementation"],
+            architecture_decisions=mock["architecture_decisions"],
+            differentiation_strategy=mock["differentiation_strategy"],
+            dataset_requirements=mock["dataset_requirements"],
+            go_to_market=mock["go_to_market"],
+            risk_assessment=mock["risk_assessment"],
+            first_90_day_milestones=mock["first_90_day_milestones"],
+            suggested_stack=mock["suggested_stack"],
+        )
+
+    def get_blueprint(self, blueprint_id: str) -> Optional[ProductBlueprint]:
+        return self.generated_blueprints.get(blueprint_id)
+
+    def list_blueprints(self) -> list[ProductBlueprint]:
+        return list(self.generated_blueprints.values())

intelligence/experiment_designer.py

@@ -0,0 +1,68 @@
+"""Experiment Designer — Autonomous Scientific Validation Engine.
+
+Follows Section 6.5: Generates minimal viable experiments to validate
+research hypotheses derived from product blueprints.
+"""
+
+import logging
+from typing import Optional
+import config
+
+logger = logging.getLogger("vectormind.intelligence")
+
+class ExperimentDesigner:
+    """Generates scientific experiment designs for research validation."""
+
+    def __init__(self):
+        self.template = """
+# Experiment Design: {technique_name} Validation
+# Generated by VectorMind Autonomous Agent
+
+## 1. Research Hypothesis
+{hypothesis}
+
+## 2. Minimal Viable Experiment (MVE)
+- Dataset: {dataset_suggestion}
+- Model Architecture: {model_suggestion}
+- Key Metric: {key_metric}
+- Target Baseline: {baseline}
+
+## 3. Implementation (PyTorch/HuggingFace)
+```python
+import torch
+import transformers
+# ... auto-generated experiment code ...
+```
+
+## 4. Pass/Fail Verdict Criteria
+The experiment is considered a PASS if the {key_metric} exceeds {baseline} 
+with a 95% confidence interval across 5 runs.
+"""
+
+    async def design_experiment(self, technique_name: str, brief: str) -> dict:
+        """Generate a structured experiment design based on a technical brief."""
+        logger.info(f"Designing experiment for: {technique_name}")
+        
+        # In a real implementation, this would call the LLM (Groq)
+        # to reason about the smallest possible validation experiment.
+        
+        hypothesis = f"Applying {technique_name} will improve state-of-the-art efficiency by at least 15%."
+        
+        experiment = {
+            "technique_name": technique_name,
+            "hypothesis": hypothesis,
+            "dataset_suggestion": "Tiny-ImageNet (Subsampled)" if "vision" in brief.lower() else "WikiText-2",
+            "model_suggestion": "MobileNet-V3" if "vision" in brief.lower() else "DistilBERT",
+            "key_metric": "Top-1 Accuracy" if "vision" in brief.lower() else "Perplexity",
+            "baseline": "0.72" if "vision" in brief.lower() else "24.5",
+            "notebook_content": self.template.format(
+                technique_name=technique_name,
+                hypothesis=hypothesis,
+                dataset_suggestion="Tiny-ImageNet" if "vision" in brief.lower() else "WikiText-2",
+                model_suggestion="MobileNet-V3" if "vision" in brief.lower() else "DistilBERT",
+                key_metric="Top-1 Accuracy" if "vision" in brief.lower() else "Perplexity",
+                baseline="0.72" if "vision" in brief.lower() else "24.5"
+            )
+        }
+        
+        return experiment

intelligence/pipeline_executor.py

@@ -0,0 +1,240 @@
+"""Pipeline execution engine.
+
+Runs generated pipeline scripts as background subprocesses, captures logs,
+and exposes runtime status/artifact locations.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Awaitable, Callable, Optional
+
+import config
+from ingestion.schema import MLPipeline
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+@dataclass
+class PipelineRun:
+    run_id: str
+    pipeline_id: str
+    status: str = "queued"  # queued|running|completed|failed|timeout
+    created_at: str = field(default_factory=_now_iso)
+    started_at: Optional[str] = None
+    finished_at: Optional[str] = None
+    exit_code: Optional[int] = None
+    duration_seconds: Optional[float] = None
+    error: str = ""
+    run_dir: str = ""
+    script_path: str = ""
+    log_path: str = ""
+    artifacts_dir: str = ""
+    command: list[str] = field(default_factory=list)
+    retry_count: int = 0
+    max_retries: int = 0
+
+    def to_dict(self) -> dict:
+        return {
+            "run_id": self.run_id,
+            "pipeline_id": self.pipeline_id,
+            "status": self.status,
+            "created_at": self.created_at,
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "exit_code": self.exit_code,
+            "duration_seconds": self.duration_seconds,
+            "error": self.error,
+            "run_dir": self.run_dir,
+            "script_path": self.script_path,
+            "log_path": self.log_path,
+            "artifacts_dir": self.artifacts_dir,
+            "command": self.command,
+            "retry_count": self.retry_count,
+            "max_retries": self.max_retries,
+        }
+
+
+class PipelineExecutor:
+    """Execute generated pipelines in managed local run directories."""
+
+    def __init__(
+        self,
+        max_concurrent_runs: int | None = None,
+        max_retries: int | None = None,
+        retry_backoff_seconds: int | None = None,
+        on_state_change: Optional[Callable[[dict], Awaitable[None] | None]] = None,
+    ):
+        self.base_dir = Path(config.DATA_DIR) / "pipeline_runs"
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        self.runs_by_pipeline: dict[str, dict[str, PipelineRun]] = {}
+        self.tasks: dict[str, asyncio.Task] = {}
+        self.max_concurrent_runs = max_concurrent_runs or config.PIPELINE_MAX_CONCURRENT_RUNS
+        self.max_retries = max_retries if max_retries is not None else config.PIPELINE_MAX_RETRIES
+        self.retry_backoff_seconds = (
+            retry_backoff_seconds
+            if retry_backoff_seconds is not None
+            else config.PIPELINE_RETRY_BACKOFF_SECONDS
+        )
+        self._semaphore = asyncio.Semaphore(max(1, self.max_concurrent_runs))
+        self.on_state_change = on_state_change
+
+    def _register_run(self, run: PipelineRun):
+        bucket = self.runs_by_pipeline.setdefault(run.pipeline_id, {})
+        bucket[run.run_id] = run
+
+    async def _emit_state(self, run: PipelineRun):
+        if not self.on_state_change:
+            return
+        result = self.on_state_change(run.to_dict())
+        if asyncio.iscoroutine(result):
+            await result
+
+    def list_runs(self, pipeline_id: str) -> list[dict]:
+        runs = list(self.runs_by_pipeline.get(pipeline_id, {}).values())
+        runs.sort(key=lambda r: r.created_at, reverse=True)
+        return [r.to_dict() for r in runs]
+
+    def get_run(self, pipeline_id: str, run_id: str) -> Optional[dict]:
+        run = self.runs_by_pipeline.get(pipeline_id, {}).get(run_id)
+        return run.to_dict() if run else None
+
+    def _prepare_run_files(self, pipeline: MLPipeline, run_id: str) -> PipelineRun:
+        run_dir = self.base_dir / pipeline.id / run_id
+        run_dir.mkdir(parents=True, exist_ok=True)
+
+        script_path = run_dir / "pipeline.py"
+        log_path = run_dir / "run.log"
+        artifacts_dir = run_dir / "artifacts"
+        artifacts_dir.mkdir(parents=True, exist_ok=True)
+
+        script_header = (
+            "import os\n"
+            "import subprocess\n"
+            f"os.environ.setdefault('VECTOR_MINDS_ARTIFACT_DIR', r'{artifacts_dir.as_posix()}')\n\n"
+        )
+        script_body = self._transpile_notebook_to_python(pipeline.notebook_content)
+        script_path.write_text(script_header + script_body, encoding="utf-8")
+
+        return PipelineRun(
+            run_id=run_id,
+            pipeline_id=pipeline.id,
+            run_dir=str(run_dir),
+            script_path=str(script_path),
+            log_path=str(log_path),
+            artifacts_dir=str(artifacts_dir),
+        )
+
+    def _transpile_notebook_to_python(self, text: str) -> str:
+        """Convert lightweight notebook magics to runnable Python."""
+        out_lines: list[str] = []
+        for raw in text.splitlines():
+            stripped = raw.lstrip()
+            indent = raw[: len(raw) - len(stripped)]
+            if stripped.startswith("!"):
+                shell_cmd = stripped[1:].strip().replace("\\", "\\\\").replace('"', '\\"')
+                out_lines.append(
+                    f'{indent}subprocess.check_call("{shell_cmd}", shell=True)'
+                )
+                continue
+            if stripped.startswith("%"):
+                out_lines.append(f"{indent}# skipped notebook magic: {stripped}")
+                continue
+            out_lines.append(raw)
+        return "\n".join(out_lines) + "\n"
+
+    async def _run_subprocess_once(self, run: PipelineRun, timeout_seconds: int):
+        run.started_at = _now_iso()
+        run.status = "running"
+        start = datetime.now(timezone.utc)
+        await self._emit_state(run)
+
+        cmd = [sys.executable, "-u", run.script_path]
+        run.command = cmd
+
+        env = os.environ.copy()
+        env["PYTHONUNBUFFERED"] = "1"
+        env["VECTOR_MINDS_RUN_ID"] = run.run_id
+        env["VECTOR_MINDS_PIPELINE_ID"] = run.pipeline_id
+        env["VECTOR_MINDS_ARTIFACT_DIR"] = run.artifacts_dir
+
+        with open(run.log_path, "w", encoding="utf-8") as log_file:
+            process = await asyncio.create_subprocess_exec(
+                *cmd,
+                cwd=run.run_dir,
+                stdout=log_file,
+                stderr=log_file,
+                env=env,
+            )
+            try:
+                await asyncio.wait_for(process.wait(), timeout=timeout_seconds)
+                run.exit_code = process.returncode
+                run.status = "completed" if process.returncode == 0 else "failed"
+            except asyncio.TimeoutError:
+                process.kill()
+                await process.wait()
+                run.exit_code = process.returncode
+                run.status = "timeout"
+                run.error = f"Execution exceeded timeout ({timeout_seconds}s)"
+
+        end = datetime.now(timezone.utc)
+        run.finished_at = _now_iso()
+        run.duration_seconds = round((end - start).total_seconds(), 3)
+        await self._emit_state(run)
+
+    async def _run_with_retry(self, run: PipelineRun, timeout_seconds: int):
+        run.max_retries = max(0, self.max_retries)
+        async with self._semaphore:
+            while True:
+                await self._run_subprocess_once(run, timeout_seconds=timeout_seconds)
+                if run.status == "completed":
+                    return
+                if run.retry_count >= run.max_retries:
+                    return
+                run.retry_count += 1
+                run.status = "queued"
+                run.error = ""
+                run.exit_code = None
+                run.started_at = None
+                run.finished_at = None
+                run.duration_seconds = None
+                await self._emit_state(run)
+                await asyncio.sleep(max(1, self.retry_backoff_seconds))
+
+    async def execute_pipeline(
+        self,
+        pipeline: MLPipeline,
+        timeout_seconds: int = 1800,
+    ) -> dict:
+        run_id = str(uuid.uuid4())
+        run = self._prepare_run_files(pipeline, run_id)
+        self._register_run(run)
+        await self._run_with_retry(run, timeout_seconds=timeout_seconds)
+        return run.to_dict()
+
+    def execute_pipeline_async(
+        self,
+        pipeline: MLPipeline,
+        timeout_seconds: int = 1800,
+    ) -> dict:
+        run_id = str(uuid.uuid4())
+        run = self._prepare_run_files(pipeline, run_id)
+        self._register_run(run)
+        run.max_retries = max(0, self.max_retries)
+
+        task = asyncio.create_task(self._run_with_retry(run, timeout_seconds=timeout_seconds))
+        self.tasks[run_id] = task
+
+        def _cleanup(_):
+            self.tasks.pop(run_id, None)
+
+        task.add_done_callback(_cleanup)
+        return run.to_dict()