diff --git a/ticker_tape.py b/ticker_tape.py index 83a3e81..f9a862e 100644 --- a/ticker_tape.py +++ b/ticker_tape.py @@ -4,6 +4,7 @@ import argparse import os import sys +import time from datetime import datetime, timedelta import requests @@ -34,8 +35,15 @@ def fetch_trades(api_key, api_secret, symbols, start, end, limit=1000, page_toke print(f"[DEBUG] GET {BASE_URL}", file=sys.stderr) print(f"[DEBUG] params: {params}", file=sys.stderr) + time.sleep(0.3) # stay under 200 req/min resp = requests.get(BASE_URL, headers=headers, params=params) + # retry once on rate limit + if resp.status_code == 429: + print("[RATE LIMITED] sleeping 5s...", file=sys.stderr) + time.sleep(5) + resp = requests.get(BASE_URL, headers=headers, params=params) + if debug: print(f"[DEBUG] HTTP {resp.status_code}", file=sys.stderr) if not resp.ok: @@ -81,10 +89,12 @@ def main(): parser = argparse.ArgumentParser(description="NYSE ticker tape via Alpaca market data") parser.add_argument("--symbols", default=DEFAULT_SYMBOLS, help=f"Comma-separated symbols (default: {DEFAULT_SYMBOLS})") + parser.add_argument("--all-nyse", metavar="FILE", + help="Read all NYSE symbols from FILE (one per line) and batch requests") parser.add_argument("--start", help="Start datetime, YYYY-MM-DD or RFC-3339 (default: previous trading day)") parser.add_argument("--end", help="End datetime (default: same day as start)") - parser.add_argument("--limit", type=int, default=1000, - help="Max trades to fetch per API call (default: 1000, max: 10000)") + parser.add_argument("--limit", type=int, default=10000, + help="Max trades to fetch per API call (default: 10000, max: 10000)") parser.add_argument("--all", action="store_true", help="Paginate through ALL trades in the range (many API calls)") parser.add_argument("--tape", default="A", @@ -121,55 +131,80 @@ def main(): tape_filter = None if args.no_filter or args.tape.lower() == "all" else args.tape.upper() + # Build symbol batches + if args.all_nyse: + with open(args.all_nyse) as f: + all_symbols = [line.strip() for line in f if line.strip()] + # Batch into groups of 500 to stay under URL length limits + batch_size = 500 + batches = [] + for i in range(0, len(all_symbols), batch_size): + batches.append(",".join(all_symbols[i:i + batch_size])) + print(f"{len(all_symbols)} symbols in {len(batches)} batches", file=sys.stderr) + # --all-nyse implies --all (paginate everything) + args.all = True + else: + batches = [args.symbols] + if args.debug: print(f"[DEBUG] start={args.start} end={args.end}", file=sys.stderr) - print(f"[DEBUG] symbols={args.symbols}", file=sys.stderr) print(f"[DEBUG] tape_filter={tape_filter} (A=NYSE-listed, B=regional, C=Nasdaq)", file=sys.stderr) - page_token = None total = 0 + api_calls = 0 exchanges_seen = {} - while True: - data = fetch_trades(api_key, api_secret, args.symbols, - args.start, args.end, args.limit, page_token, debug=args.debug) - trades = flatten_and_sort(data) - if args.debug: + for batch_num, symbols in enumerate(batches, 1): + if len(batches) > 1: + print(f"Batch {batch_num}/{len(batches)}...", file=sys.stderr) + + page_token = None + while True: + data = fetch_trades(api_key, api_secret, symbols, + args.start, args.end, args.limit, page_token, debug=args.debug) + api_calls += 1 + trades = flatten_and_sort(data) + + if args.debug: + for trade in trades: + ex = trade.get("x", "?") + tape = trade.get("z", "?") + exchanges_seen[ex] = exchanges_seen.get(ex, 0) + 1 + exchanges_seen[f"tape:{tape}"] = exchanges_seen.get(f"tape:{tape}", 0) + 1 + for trade in trades: - ex = trade.get("x", "?") - tape = trade.get("z", "?") - exchanges_seen[ex] = exchanges_seen.get(ex, 0) + 1 - exchanges_seen[f"tape:{tape}"] = exchanges_seen.get(f"tape:{tape}", 0) + 1 + if tape_filter and trade.get("z") != tape_filter: + continue + if trade["s"] < 100: + continue - for trade in trades: - if tape_filter and trade.get("z") != tape_filter: - continue - if trade["s"] < 100: - continue + lots = trade["s"] // 100 - lots = trade["s"] // 100 + if args.raw: + print(f"{trade['t']},{trade['sym']},{lots},{trade['p']:.2f}") + else: + ts = trade["t"][:19].replace("T", " ") + line = format_tape_line(trade["sym"], lots, trade["p"]) + print(f"{ts} {line}") + total += 1 - if args.raw: - print(f"{trade['t']},{trade['sym']},{lots},{trade['p']:.2f}") - else: - ts = trade["t"][:19].replace("T", " ") - line = format_tape_line(trade["sym"], lots, trade["p"]) - print(f"{ts} {line}") - total += 1 + page_token = data.get("next_page_token") + if not page_token or not args.all: + break - page_token = data.get("next_page_token") - if not page_token or not args.all: - break + if api_calls % 100 == 0: + print(f" {api_calls} API calls, {total} trades so far...", file=sys.stderr) if args.debug and exchanges_seen: print(f"[DEBUG] exchange codes seen: {exchanges_seen}", file=sys.stderr) print(f"[DEBUG] (exchanges: N=NYSE, Q=Nasdaq, P=Arca, D=FINRA, K=EDGX...)", file=sys.stderr) print(f"[DEBUG] (tapes: A=NYSE-listed, B=regional, C=Nasdaq-listed)", file=sys.stderr) + print(f"Done: {total} trades, {api_calls} API calls.", file=sys.stderr) if total == 0: print("No trades found. Try --no-filter or different --tape/symbols/dates.", file=sys.stderr) elif not args.all and data.get("next_page_token"): - print(f"\n--- {total} trades shown. More available; use --all to paginate. ---", file=sys.stderr) + print(f"More available; use --all to paginate.", file=sys.stderr) if __name__ == "__main__":