template/.gitea/workflows/record-to-qdrant.yml

name: Record to Qdrant

on:
  push:
    branches: ['**']
  pull_request:
    types: [opened, closed, synchronize]
  create:
  delete:
  release:
    types: [published]
  workflow_dispatch:
  schedule:
    - cron: '0 15 * * *'

jobs:
  record-event:
    if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
    runs-on: ubuntu-latest
    container:
      image: python:3.11-slim
    defaults:
      run:
        shell: bash
    steps:
      - name: Install dependencies
        run: apt-get update -qq && apt-get install -y -qq curl bash git

      - name: Checkout
        run: |
          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo

      - name: Record event to Qdrant
        env:
          EVENT: ${{ github.event_name }}
          REPO: ${{ github.repository }}
          BRANCH: ${{ github.ref_name }}
          ACTOR: ${{ github.actor }}
          RUN_ID: ${{ github.run_id }}
          COMMIT: ${{ github.sha }}
        run: |
          COMMIT_SHORT=$(echo "${COMMIT}" | cut -c1-8)
          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
          CONTENT="[${EVENT}] ${REPO} @ ${BRANCH} by ${ACTOR} | commit: ${COMMIT_SHORT} | run: ${RUN_ID} | time: ${TIMESTAMP}"
          echo "Content: ${CONTENT}"
          echo -n "${CONTENT}" > /tmp/content.txt

          python3 -c "import json;open('/tmp/embed_req.json','w').write(json.dumps({'model':'bge-m3:latest','input':open('/tmp/content.txt').read()}))"

          curl -s -X POST http://192.168.11.124:11434/api/embed \
            -H "Content-Type: application/json" \
            --data-binary @/tmp/embed_req.json > /tmp/embed_response.json

          export TIMESTAMP RUN_ID EVENT REPO BRANCH ACTOR
          python3 /tmp/repo/scripts/record_event.py

          curl -s -X PUT http://192.168.11.124:6333/collections/gitea_actions/points \
            -H "Content-Type: application/json" \
            --data-binary @/tmp/payload.json
          echo "Done"

  nightly-digest:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    container:
      image: python:3.11-slim
    defaults:
      run:
        shell: bash
    steps:
      - name: Install dependencies
        run: apt-get update -qq && apt-get install -y -qq curl bash git

      - name: Checkout
        run: |
          git clone --depth 1 http://admin:0ebeb908eb6bc62c20d4d0cb937970651859f409@10.43.165.66:3000/k3s-manifests/pod-manifests.git /tmp/repo

      - name: Summarize and store digest
        run: |
          TODAY=$(date -u +"%Y-%m-%d")
          TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")

          curl -s -X POST http://192.168.11.124:6333/collections/gitea_actions/points/scroll \
            -H "Content-Type: application/json" \
            --data-binary "{\"limit\":100,\"with_payload\":true,\"with_vector\":false,\"filter\":{\"must\":[{\"key\":\"timestamp\",\"range\":{\"gte\":\"${TODAY}T00:00:00Z\"}}]}}" \
            > /tmp/scroll.json

          COUNT=$(python3 -c "import json;d=json.load(open('/tmp/scroll.json'));print(len(d.get('result',{}).get('points',[])))")
          echo "Records today: ${COUNT}"
          if [ "${COUNT}" = "0" ]; then echo "No records, skipping."; exit 0; fi

          python3 /tmp/repo/scripts/extract_texts.py

          TODAY=${TODAY} python3 /tmp/repo/scripts/build_gen_payload.py

          curl -s -X POST http://192.168.11.124:11434/api/generate \
            -H "Content-Type: application/json" \
            --data-binary @/tmp/gen_payload.json > /tmp/gen_response.json

          python3 -c "import json;open('/tmp/summary.txt','w').write(json.load(open('/tmp/gen_response.json')).get('response',''))"
          echo "Summary: $(cat /tmp/summary.txt | head -c 100)"

          python3 /tmp/repo/scripts/build_embed_input.py

          curl -s -X POST http://192.168.11.124:11434/api/embed \
            -H "Content-Type: application/json" \
            --data-binary @/tmp/embed_input.json > /tmp/embed_response.json

          TODAY=${TODAY} TIMESTAMP=${TIMESTAMP} COUNT=${COUNT} python3 /tmp/repo/scripts/build_digest_payload.py

          RESULT=$(curl -s -X PUT http://192.168.11.124:6333/collections/llm_conversations_digest/points \
            -H "Content-Type: application/json" \
            --data-binary @/tmp/digest_payload.json)
          echo "Digest result: ${RESULT}"