gets all the nyse
This commit is contained in:
parent
376ab0d395
commit
215c0de21c
1 changed files with 65 additions and 30 deletions
|
|
@ -4,6 +4,7 @@
|
|||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import requests
|
||||
|
|
@ -34,8 +35,15 @@ def fetch_trades(api_key, api_secret, symbols, start, end, limit=1000, page_toke
|
|||
print(f"[DEBUG] GET {BASE_URL}", file=sys.stderr)
|
||||
print(f"[DEBUG] params: {params}", file=sys.stderr)
|
||||
|
||||
time.sleep(0.3) # stay under 200 req/min
|
||||
resp = requests.get(BASE_URL, headers=headers, params=params)
|
||||
|
||||
# retry once on rate limit
|
||||
if resp.status_code == 429:
|
||||
print("[RATE LIMITED] sleeping 5s...", file=sys.stderr)
|
||||
time.sleep(5)
|
||||
resp = requests.get(BASE_URL, headers=headers, params=params)
|
||||
|
||||
if debug:
|
||||
print(f"[DEBUG] HTTP {resp.status_code}", file=sys.stderr)
|
||||
if not resp.ok:
|
||||
|
|
@ -81,10 +89,12 @@ def main():
|
|||
parser = argparse.ArgumentParser(description="NYSE ticker tape via Alpaca market data")
|
||||
parser.add_argument("--symbols", default=DEFAULT_SYMBOLS,
|
||||
help=f"Comma-separated symbols (default: {DEFAULT_SYMBOLS})")
|
||||
parser.add_argument("--all-nyse", metavar="FILE",
|
||||
help="Read all NYSE symbols from FILE (one per line) and batch requests")
|
||||
parser.add_argument("--start", help="Start datetime, YYYY-MM-DD or RFC-3339 (default: previous trading day)")
|
||||
parser.add_argument("--end", help="End datetime (default: same day as start)")
|
||||
parser.add_argument("--limit", type=int, default=1000,
|
||||
help="Max trades to fetch per API call (default: 1000, max: 10000)")
|
||||
parser.add_argument("--limit", type=int, default=10000,
|
||||
help="Max trades to fetch per API call (default: 10000, max: 10000)")
|
||||
parser.add_argument("--all", action="store_true",
|
||||
help="Paginate through ALL trades in the range (many API calls)")
|
||||
parser.add_argument("--tape", default="A",
|
||||
|
|
@ -121,55 +131,80 @@ def main():
|
|||
|
||||
tape_filter = None if args.no_filter or args.tape.lower() == "all" else args.tape.upper()
|
||||
|
||||
# Build symbol batches
|
||||
if args.all_nyse:
|
||||
with open(args.all_nyse) as f:
|
||||
all_symbols = [line.strip() for line in f if line.strip()]
|
||||
# Batch into groups of 500 to stay under URL length limits
|
||||
batch_size = 500
|
||||
batches = []
|
||||
for i in range(0, len(all_symbols), batch_size):
|
||||
batches.append(",".join(all_symbols[i:i + batch_size]))
|
||||
print(f"{len(all_symbols)} symbols in {len(batches)} batches", file=sys.stderr)
|
||||
# --all-nyse implies --all (paginate everything)
|
||||
args.all = True
|
||||
else:
|
||||
batches = [args.symbols]
|
||||
|
||||
if args.debug:
|
||||
print(f"[DEBUG] start={args.start} end={args.end}", file=sys.stderr)
|
||||
print(f"[DEBUG] symbols={args.symbols}", file=sys.stderr)
|
||||
print(f"[DEBUG] tape_filter={tape_filter} (A=NYSE-listed, B=regional, C=Nasdaq)", file=sys.stderr)
|
||||
|
||||
page_token = None
|
||||
total = 0
|
||||
api_calls = 0
|
||||
exchanges_seen = {}
|
||||
while True:
|
||||
data = fetch_trades(api_key, api_secret, args.symbols,
|
||||
args.start, args.end, args.limit, page_token, debug=args.debug)
|
||||
trades = flatten_and_sort(data)
|
||||
|
||||
if args.debug:
|
||||
for batch_num, symbols in enumerate(batches, 1):
|
||||
if len(batches) > 1:
|
||||
print(f"Batch {batch_num}/{len(batches)}...", file=sys.stderr)
|
||||
|
||||
page_token = None
|
||||
while True:
|
||||
data = fetch_trades(api_key, api_secret, symbols,
|
||||
args.start, args.end, args.limit, page_token, debug=args.debug)
|
||||
api_calls += 1
|
||||
trades = flatten_and_sort(data)
|
||||
|
||||
if args.debug:
|
||||
for trade in trades:
|
||||
ex = trade.get("x", "?")
|
||||
tape = trade.get("z", "?")
|
||||
exchanges_seen[ex] = exchanges_seen.get(ex, 0) + 1
|
||||
exchanges_seen[f"tape:{tape}"] = exchanges_seen.get(f"tape:{tape}", 0) + 1
|
||||
|
||||
for trade in trades:
|
||||
ex = trade.get("x", "?")
|
||||
tape = trade.get("z", "?")
|
||||
exchanges_seen[ex] = exchanges_seen.get(ex, 0) + 1
|
||||
exchanges_seen[f"tape:{tape}"] = exchanges_seen.get(f"tape:{tape}", 0) + 1
|
||||
if tape_filter and trade.get("z") != tape_filter:
|
||||
continue
|
||||
if trade["s"] < 100:
|
||||
continue
|
||||
|
||||
for trade in trades:
|
||||
if tape_filter and trade.get("z") != tape_filter:
|
||||
continue
|
||||
if trade["s"] < 100:
|
||||
continue
|
||||
lots = trade["s"] // 100
|
||||
|
||||
lots = trade["s"] // 100
|
||||
if args.raw:
|
||||
print(f"{trade['t']},{trade['sym']},{lots},{trade['p']:.2f}")
|
||||
else:
|
||||
ts = trade["t"][:19].replace("T", " ")
|
||||
line = format_tape_line(trade["sym"], lots, trade["p"])
|
||||
print(f"{ts} {line}")
|
||||
total += 1
|
||||
|
||||
if args.raw:
|
||||
print(f"{trade['t']},{trade['sym']},{lots},{trade['p']:.2f}")
|
||||
else:
|
||||
ts = trade["t"][:19].replace("T", " ")
|
||||
line = format_tape_line(trade["sym"], lots, trade["p"])
|
||||
print(f"{ts} {line}")
|
||||
total += 1
|
||||
page_token = data.get("next_page_token")
|
||||
if not page_token or not args.all:
|
||||
break
|
||||
|
||||
page_token = data.get("next_page_token")
|
||||
if not page_token or not args.all:
|
||||
break
|
||||
if api_calls % 100 == 0:
|
||||
print(f" {api_calls} API calls, {total} trades so far...", file=sys.stderr)
|
||||
|
||||
if args.debug and exchanges_seen:
|
||||
print(f"[DEBUG] exchange codes seen: {exchanges_seen}", file=sys.stderr)
|
||||
print(f"[DEBUG] (exchanges: N=NYSE, Q=Nasdaq, P=Arca, D=FINRA, K=EDGX...)", file=sys.stderr)
|
||||
print(f"[DEBUG] (tapes: A=NYSE-listed, B=regional, C=Nasdaq-listed)", file=sys.stderr)
|
||||
|
||||
print(f"Done: {total} trades, {api_calls} API calls.", file=sys.stderr)
|
||||
if total == 0:
|
||||
print("No trades found. Try --no-filter or different --tape/symbols/dates.", file=sys.stderr)
|
||||
elif not args.all and data.get("next_page_token"):
|
||||
print(f"\n--- {total} trades shown. More available; use --all to paginate. ---", file=sys.stderr)
|
||||
print(f"More available; use --all to paginate.", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue