Debug API Changes — How to Compare API Responses to Find Breaking Changes

# Step 1: Capture baseline response (before the change)
curl -s https://api.example.com/users/123 \
  -H "Authorization: Bearer TOKEN" \
  | python3 -m json.tool > baseline.json

# Step 2: After the API change, capture new response
curl -s https://api.example.com/users/123 \
  -H "Authorization: Bearer TOKEN" \
  | python3 -m json.tool > current.json

# Step 3: Compare with diff (shows line-level changes)
diff baseline.json current.json

# Better: use jq for normalized comparison (sorts keys, ignores formatting)
diff <(jq -S . baseline.json) <(jq -S . current.json)

# Colored diff output (easier to read)
diff --color=always <(jq -S . baseline.json) <(jq -S . current.json)

# Find only key changes (added/removed fields, not value changes):
diff <(jq -r '[paths | join(".")]' baseline.json | sort) \
     <(jq -r '[paths | join(".")]' current.json | sort)

# Compare specific nested field:
jq '.user.address' baseline.json > /tmp/old_addr.json
jq '.user.address' current.json > /tmp/new_addr.json
diff /tmp/old_addr.json /tmp/new_addr.json

# Batch compare all endpoints in a list:
for endpoint in /users /products /orders; do
  curl -s "https://api.example.com$endpoint" | jq -S . > "current_$(echo $endpoint | tr / _).json"
  diff "baseline_$(echo $endpoint | tr / _).json" "current_$(echo $endpoint | tr / _).json" \
    && echo "$endpoint: no changes" \
    || echo "$endpoint: CHANGED ⚠️"
done

import jsonschema
import requests
import json
from typing import Any

# Define expected schema (generate from a known-good response with genson:
# pip install genson; python -c "from genson import SchemaBuilder; ...")
expected_schema = {
    "type": "object",
    "required": ["id", "name", "email", "created_at"],
    "properties": {
        "id": {"type": "integer"},
        "name": {"type": "string", "minLength": 1},
        "email": {"type": "string", "format": "email"},
        "created_at": {"type": "string", "pattern": r"^d{4}-d{2}-d{2}"},
        "role": {"type": "string", "enum": ["admin", "user", "viewer"]},
        "tags": {"type": "array", "items": {"type": "string"}},
        "metadata": {"type": ["object", "null"]},
    },
    "additionalProperties": False,  # Alert on new unknown fields
}

def check_api_contract(url: str, token: str) -> bool:
    response = requests.get(url, headers={"Authorization": f"Bearer {token}"})
    response.raise_for_status()
    data = response.json()

    try:
        jsonschema.validate(data, expected_schema)
        print(f"✅ {url} — matches expected schema")
        return True
    except jsonschema.ValidationError as e:
        print(f"❌ {url} — Schema violation:")
        print(f"   Error: {e.message}")
        print(f"   Path: {' → '.join(str(p) for p in e.path)}")
        print(f"   Got value: {e.instance!r}")
        return False
    except jsonschema.SchemaError as e:
        print(f"⚠️  Schema definition error: {e.message}")
        return False

# Run against multiple endpoints
endpoints = [
    "https://api.example.com/users/1",
    "https://api.example.com/users/2",
    "https://api.example.com/users/100",  # edge: near boundary
]

all_pass = all(check_api_contract(url, "mytoken") for url in endpoints)
print("\nAll checks passed!" if all_pass else "\n⚠️  Contract violations detected!")

from typing import Any

def semantic_diff(old_data: Any, new_data: Any, path: str = "") -> list[str]:
    """Recursively compare JSON objects and report meaningful differences.
    Detects: type changes, added/removed fields, value changes."""
    issues = []

    # Type change is most critical — catches int→string, null→string, etc.
    if type(old_data) != type(new_data):
        issues.append(
            f"TYPE CHANGE at '{path}': "
            f"{type(old_data).__name__} → {type(new_data).__name__} "
            f"(was: {old_data!r}, now: {new_data!r})"
        )
        return issues  # Don't recurse — types differ fundamentally

    if isinstance(old_data, dict):
        old_keys, new_keys = set(old_data), set(new_data)

        # Removed keys (breaking!)
        for key in old_keys - new_keys:
            issues.append(f"REMOVED FIELD: '{path}.{key}' (was: {old_data[key]!r})")

        # Added keys (usually non-breaking, but track them)
        for key in new_keys - old_keys:
            issues.append(f"ADDED FIELD: '{path}.{key}' = {new_data[key]!r}")

        # Check common keys recursively
        for key in old_keys & new_keys:
            child_path = f"{path}.{key}" if path else key
            issues.extend(semantic_diff(old_data[key], new_data[key], child_path))

    elif isinstance(old_data, list):
        # Compare first elements as schema examples
        if old_data and new_data:
            issues.extend(semantic_diff(old_data[0], new_data[0], f"{path}[0]"))
        if bool(old_data) != bool(new_data):
            issues.append(f"EMPTINESS CHANGE at '{path}': was {'empty' if not old_data else 'non-empty'}")
    else:
        # Scalar value change — note but may be expected
        if old_data != new_data:
            issues.append(f"VALUE CHANGE at '{path}': {old_data!r} → {new_data!r}")

    return issues

# Usage: compare baseline with current
import requests, json

baseline = json.loads(open("baseline.json").read())
current = requests.get("https://api.example.com/users/1",
                       headers={"Authorization": "Bearer TOKEN"}).json()

issues = semantic_diff(baseline, current)
breaking = [i for i in issues if any(w in i for w in ["REMOVED", "TYPE CHANGE"])]
non_breaking = [i for i in issues if i not in breaking]

print(f"🔴 Breaking changes ({len(breaking)}):")
for issue in breaking: print(f"  {issue}")

print(f"\n🟡 Non-breaking changes ({len(non_breaking)}):")
for issue in non_breaking: print(f"  {issue}")

name: API Contract Tests
on:
  push:
    branches: [main, develop]
  pull_request:

jobs:
  api-contract:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pip install requests jsonschema

      - name: Run API schema validation
        run: python tests/validate_api_schema.py
        env:
          API_URL: https://staging.api.example.com
          API_TOKEN: ${{ secrets.TEST_TOKEN }}

      - name: Compare against baseline
        run: |
          # Fetch current response
          curl -s https://staging.api.example.com/users/1 \
            -H "Authorization: Bearer ${{ secrets.TEST_TOKEN }}" \
            | jq -S . > current.json

          # Diff against committed baseline
          diff tests/baselines/users_1.json current.json
          if [ $? -ne 0 ]; then
            echo "::error::API response changed — update baseline or fix regression"
            exit 1
          fi

      - name: OpenAPI contract test (dredd)
        run: |
          npm install -g dredd
          dredd openapi.yaml https://staging.api.example.com \
            --header "Authorization:Bearer ${{ secrets.TEST_TOKEN }}" \
            --reporter junit --output dredd-results.xml

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-results
          path: dredd-results.xml