portfolio_analyzer/
history.rs

1//! # Dated daily history for research
2//!
3//! The live fetcher (`fetcher.rs`) throws timestamps away because the live
4//! analyzers only ever look at "the latest bar".  A backtest cannot do that:
5//! to split *train* from *test* honestly we need to know **when** every bar
6//! happened.  This module downloads long daily histories from Yahoo, keeps
7//! the date, and stores dividend-and-split adjusted OHLC so that a $1
8//! dividend never looks like a $1 crash to the indicator.
9//!
10//! File format (`data/history/SYMBOL.csv`):
11//!
12//! ```text
13//! date,open,high,low,close,volume
14//! 2004-01-02,10.21,10.40,10.11,10.33,123400
15//! ```
16//!
17//! The extra `date` column is ignored by `fetcher::load_ohlcv_csv`, so these
18//! files also work with every older tool in the crate.
19
20use crate::indicators::OHLCV;
21use anyhow::{anyhow, Context, Result};
22use chrono::{DateTime, NaiveDate};
23use reqwest::blocking::Client;
24use serde_json::Value;
25use std::fs;
26use std::path::Path;
27use std::time::Duration;
28
29/// One daily bar plus the calendar date it closed on.
30#[derive(Debug, Clone)]
31pub struct DatedBar {
32    pub date: NaiveDate,
33    pub bar: OHLCV,
34}
35
36/// Download `range` (e.g. `"25y"` or `"max"`) of daily bars for `symbol`.
37///
38/// OHLC are multiplied by `adjclose / close` for each bar, which is the
39/// standard way to back-adjust for dividends and splits.  Bars with any
40/// missing field are skipped rather than forward-filled: a fabricated bar is
41/// worse than a missing one for a squeeze detector.
42pub fn fetch_daily(client: &Client, symbol: &str, range: &str) -> Result<Vec<DatedBar>> {
43    let url = format!(
44        "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?interval=1d&range={range}&includeAdjustedClose=true"
45    );
46    let mut last_err = anyhow!("no attempts");
47    for attempt in 0..4u32 {
48        if attempt > 0 {
49            std::thread::sleep(Duration::from_millis(750 * 2u64.pow(attempt)));
50        }
51        let response = match client
52            .get(&url)
53            .header("User-Agent", "Mozilla/5.0")
54            .timeout(Duration::from_secs(30))
55            .send()
56        {
57            Ok(response) => response,
58            Err(error) => {
59                last_err = anyhow!("http: {error}");
60                continue;
61            }
62        };
63        if response.status().as_u16() == 429 {
64            last_err = anyhow!("rate limited");
65            std::thread::sleep(Duration::from_secs(5));
66            continue;
67        }
68        let body: Value = match response.json() {
69            Ok(body) => body,
70            Err(error) => {
71                last_err = anyhow!("json: {error}");
72                continue;
73            }
74        };
75        return parse(&body);
76    }
77    Err(last_err)
78}
79
80fn parse(body: &Value) -> Result<Vec<DatedBar>> {
81    let result = &body["chart"]["result"][0];
82    let timestamps = result["timestamp"]
83        .as_array()
84        .ok_or_else(|| anyhow!("no timestamps"))?;
85    let quote = &result["indicators"]["quote"][0];
86    let adj = result["indicators"]["adjclose"][0]["adjclose"].as_array();
87    let field = |name: &str| quote[name].as_array().cloned().unwrap_or_default();
88    let (open, high, low, close, volume) = (
89        field("open"),
90        field("high"),
91        field("low"),
92        field("close"),
93        field("volume"),
94    );
95    let mut bars = Vec::with_capacity(timestamps.len());
96    for (i, ts) in timestamps.iter().enumerate() {
97        let get = |v: &Vec<Value>| v.get(i).and_then(Value::as_f64);
98        let (Some(o), Some(h), Some(l), Some(c)) = (get(&open), get(&high), get(&low), get(&close))
99        else {
100            continue;
101        };
102        if c <= 0.0 || h < l {
103            continue;
104        }
105        let ratio = adj
106            .and_then(|a| a.get(i))
107            .and_then(Value::as_f64)
108            .map(|a| a / c)
109            .filter(|r| r.is_finite() && *r > 0.0)
110            .unwrap_or(1.0);
111        let Some(date) = ts
112            .as_i64()
113            .and_then(|s| DateTime::from_timestamp(s, 0))
114            .map(|d| d.date_naive())
115        else {
116            continue;
117        };
118        bars.push(DatedBar {
119            date,
120            bar: OHLCV {
121                open: o * ratio,
122                high: h * ratio,
123                low: l * ratio,
124                close: c * ratio,
125                volume: get(&volume).unwrap_or(0.0),
126            },
127        });
128    }
129    // Yahoo occasionally repeats the final (live) bar; keep the last copy.
130    bars.dedup_by(|later, earlier| {
131        if later.date == earlier.date {
132            *earlier = later.clone();
133            true
134        } else {
135            false
136        }
137    });
138    Ok(bars)
139}
140
141/// Write bars as `date,open,high,low,close,volume`.
142pub fn save(path: &Path, bars: &[DatedBar]) -> Result<()> {
143    if let Some(parent) = path.parent() {
144        fs::create_dir_all(parent)?;
145    }
146    let mut out = String::from("date,open,high,low,close,volume\n");
147    for b in bars {
148        out.push_str(&format!(
149            "{},{:.6},{:.6},{:.6},{:.6},{}\n",
150            b.date, b.bar.open, b.bar.high, b.bar.low, b.bar.close, b.bar.volume
151        ));
152    }
153    fs::write(path, out).with_context(|| format!("writing {}", path.display()))
154}
155
156/// Read a file written by [`save`].
157pub fn load(path: &Path) -> Result<Vec<DatedBar>> {
158    let text = fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
159    let mut bars = Vec::new();
160    for line in text.lines().skip(1) {
161        let f: Vec<&str> = line.split(',').collect();
162        if f.len() < 6 {
163            continue;
164        }
165        let num = |i: usize| f[i].parse::<f64>().unwrap_or(f64::NAN);
166        let Ok(date) = NaiveDate::parse_from_str(f[0], "%Y-%m-%d") else {
167            continue;
168        };
169        let bar = OHLCV {
170            open: num(1),
171            high: num(2),
172            low: num(3),
173            close: num(4),
174            volume: num(5),
175        };
176        if [bar.open, bar.high, bar.low, bar.close]
177            .iter()
178            .all(|v| v.is_finite() && *v > 0.0)
179        {
180            bars.push(DatedBar { date, bar });
181        }
182    }
183    Ok(bars)
184}
185
186/// How many *real* dollars one adjusted dollar was worth on each date.
187///
188/// The research bars are dividend-and-split adjusted, which is right for an
189/// indicator but wrong for account sizing: an option contract on 2003 AAPL
190/// cost what 2003 AAPL actually traded at, not its adjusted $0.30.  Yahoo's
191/// `close` undoes dividends but keeps splits; multiplying back every split
192/// that happened later gives the price printed on that day.
193/// `real price = adjusted price × scale`.
194pub fn fetch_price_scale(client: &Client, symbol: &str, range: &str) -> Result<Vec<(NaiveDate, f64)>> {
195    let url = format!(
196        "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}?interval=1d&range={range}&includeAdjustedClose=true&events=split"
197    );
198    let body: Value = client
199        .get(&url)
200        .header("User-Agent", "Mozilla/5.0")
201        .timeout(Duration::from_secs(30))
202        .send()?
203        .json()?;
204    let result = &body["chart"]["result"][0];
205    let ts = result["timestamp"].as_array().ok_or_else(|| anyhow!("no timestamps"))?;
206    let close = result["indicators"]["quote"][0]["close"].as_array().cloned().unwrap_or_default();
207    let adj = result["indicators"]["adjclose"][0]["adjclose"].as_array().cloned().unwrap_or_default();
208    let mut splits: Vec<(i64, f64)> = result["events"]["splits"]
209        .as_object()
210        .map(|m| {
211            m.values()
212                .filter_map(|s| Some((s["date"].as_i64()?, s["numerator"].as_f64()? / s["denominator"].as_f64()?)))
213                .filter(|(_, r)| r.is_finite() && *r > 0.0)
214                .collect()
215        })
216        .unwrap_or_default();
217    splits.sort_by_key(|s| s.0);
218    let mut out = Vec::new();
219    for (i, t) in ts.iter().enumerate() {
220        let (Some(t), Some(c), Some(a)) = (t.as_i64(), close.get(i).and_then(Value::as_f64), adj.get(i).and_then(Value::as_f64)) else { continue };
221        if c <= 0.0 || a <= 0.0 {
222            continue;
223        }
224        let later: f64 = splits.iter().filter(|(d, _)| *d > t).map(|(_, r)| r).product();
225        let Some(date) = DateTime::from_timestamp(t, 0).map(|d| d.date_naive()) else { continue };
226        out.push((date, c * later / a));
227    }
228    out.dedup_by(|b, a| {
229        if a.0 == b.0 {
230            *a = *b;
231            true
232        } else {
233            false
234        }
235    });
236    Ok(out)
237}
238
239/// Save / load `date,scale` files written for [`fetch_price_scale`].
240pub fn save_scale(path: &Path, rows: &[(NaiveDate, f64)]) -> Result<()> {
241    if let Some(parent) = path.parent() {
242        fs::create_dir_all(parent)?;
243    }
244    let mut out = String::from("date,scale\n");
245    for (d, s) in rows {
246        out.push_str(&format!("{d},{s:.8}\n"));
247    }
248    fs::write(path, out).with_context(|| format!("writing {}", path.display()))
249}
250
251pub fn load_scale(path: &Path) -> Result<Vec<(NaiveDate, f64)>> {
252    let text = fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
253    Ok(text
254        .lines()
255        .skip(1)
256        .filter_map(|l| {
257            let (d, s) = l.split_once(',')?;
258            Some((NaiveDate::parse_from_str(d, "%Y-%m-%d").ok()?, s.parse().ok()?))
259        })
260        .collect())
261}
portfolio_analyzer/history.rs

portfolio_analyzer/
history.rs
