Real-time updates

WATERMARK="2024-01-01T00:00:00Z"

CURSOR=""
MAX_SEEN=""
while :; do
  URL="https://nordicfinancialnews.com/api/v1/articles?updated_after=$WATERMARK&limit=100&fields=id,title,updated_at"
  [ -n "$CURSOR" ] && URL="$URL&cursor=$CURSOR"

  RESPONSE=$(curl -s -H "Authorization: Bearer $API_KEY" "$URL")
  echo "$RESPONSE" | jq -c '.articles[]'   # upsert by id

  # Records arrive in ascending updated_at order, so the last one is the highest.
  LAST=$(echo "$RESPONSE" | jq -r '.articles[-1].updated_at // empty')
  [ -n "$LAST" ] && MAX_SEEN="$LAST"

  CURSOR=$(echo "$RESPONSE" | jq -r '.pagination.next_cursor // empty')
  [ -z "$CURSOR" ] && break
done

# Commit the high-water mark minus a one-minute overlap. GNU date shown;
# on macOS use: date -u -j -f "%Y-%m-%dT%H:%M:%S" "${MAX_SEEN%%.*}" -v-1M +"%Y-%m-%dT%H:%M:%SZ"
if [ -n "$MAX_SEEN" ]; then
  WATERMARK=$(date -u -d "$MAX_SEEN - 1 minute" +"%Y-%m-%dT%H:%M:%SZ")
fi

import requests
import time
from datetime import datetime, timedelta, timezone

API_KEY = "your_api_key"
BASE = "https://nordicfinancialnews.com/api/v1"
headers = {"Authorization": f"Bearer {API_KEY}"}

OVERLAP = timedelta(minutes=1)
ENDPOINTS = ["articles", "stories", "calendar_events"]

# Each endpoint has its own change stream, so each needs its own watermark.
watermarks = {endpoint: "2024-01-01T00:00:00Z" for endpoint in ENDPOINTS}


def drain(endpoint, updated_after):
    """Walk every page of changed records. Returns the highest updated_at seen, or None."""
    cursor = None
    high_water = None

    while True:
        params = {"updated_after": updated_after, "limit": 100}
        if cursor:
            params["cursor"] = cursor

        resp = requests.get(f"{BASE}/{endpoint}", headers=headers, params=params)
        resp.raise_for_status()
        body = resp.json()

        for item in body[endpoint]:
            upsert(item)  # reconcile by item["id"]; re-delivery is expected

            seen = datetime.fromisoformat(item["updated_at"].replace("Z", "+00:00"))
            if high_water is None or seen > high_water:
                high_water = seen

        cursor = body["pagination"]["next_cursor"]
        if cursor is None:
            return high_water


while True:
    for endpoint in ENDPOINTS:
        high_water = drain(endpoint, watermarks[endpoint])

        # No records changed: keep the existing watermark rather than advancing blindly.
        if high_water is not None:
            watermarks[endpoint] = (high_water - OVERLAP).strftime("%Y-%m-%dT%H:%M:%SZ")

    time.sleep(60)  # poll every 60 seconds