// Copyright (c) 2014 panacoran <panacoran@users.sourceforge.jp>
// This program is part of OmegaChart.
// OmegaChart is licensed under the Apache License, Version 2.0.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;
using Zanetti.Data;

namespace Zanetti.DataSource.Specialized
{
    internal class YahooDataSource : DailyDataSource
    {
        private readonly Object _syncObject = new object();
        private readonly List<int> _codes = new List<int>();
        private Queue<int> _codeQueue;
        private readonly List<int> _series = new List<int>();
        private readonly Queue<FetchResult> _resultQueue = new Queue<FetchResult>();
        private bool _terminate;
        private Exception _exception;
        private const int DaysAtOnce = 50; // 一度に取得する時系列の営業日数

        private class FetchResult
        {
            public enum Status
            {
                Success,
                Failure,
                Obsolete,
                Retry,
            }

            public int Code;
            public SortedDictionary<int, NewDailyData> Prices;
            public Status ReturnStatus;
        }

        public YahooDataSource(int[] dates) : base(dates)
        {
            foreach (AbstractBrand brand in Env.BrandCollection.Values)
            {
                var basic = brand as BasicBrand;
                if (brand.Market == MarketType.B || brand.Market == MarketType.Custom ||
                    basic == null || basic.Obsolete)
                    continue;
                _codes.Add(brand.Code);
            }
        }

        public override int TotalStep
        {
            get { return (_codes.Count + 2) * ((_dates.Length + DaysAtOnce - 1) / DaysAtOnce); } // +2はNikkei225とTOPIX
        }

        public override void Run()
        {
            var threads = new Thread[2];
            for (var i = 0; i < threads.Length; i++)
                (threads[i] = new Thread(RunFetchPrices) {Name = "Fetch Thread " + i}).Start();
            var dates = new List<int>(_dates);
            try
            {
                do
                {
                    // 日経平均の時系列データの存在を確認する。
                    var n = Math.Min(DaysAtOnce, dates.Count);
                    var original = dates.GetRange(0, n);
                    var nikkei225 = FetchPrices((int)BuiltInIndex.Nikkei225, original);
                    if (nikkei225.ReturnStatus != FetchResult.Status.Success)
                        throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}～{1}",
                                                          original[0], original[original.Count - 1]));
                    dates.RemoveRange(0, n);
                    _series.Clear();
                    foreach (var date in original)
                    {
                        if (nikkei225.Prices[date].close == 0)
                            nikkei225.Prices.Remove(date);
                        else
                            _series.Add(date);
                    }
                    if (_series.Count == 0)
                        return;
                    UpdateDataFarm((int)BuiltInIndex.Nikkei225, nikkei225.Prices);
                    SendMessage(AsyncConst.WM_ASYNCPROCESS, (int)BuiltInIndex.Nikkei225, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                    _codeQueue = new Queue<int>(_codes);
                    _codeQueue.Enqueue((int)BuiltInIndex.TOPIX);
                    var retry = 0;
                    while (true)
                    {
                        int numCodes;
                        lock (_syncObject)
                        {
                            numCodes = _codeQueue.Count;
                            Monitor.PulseAll(_syncObject);
                        }
                        for (var i = 0; i < numCodes; i++)
                        {
                            FetchResult result;
                            lock (_resultQueue)
                            {
                                while (_resultQueue.Count == 0 && _exception == null)
                                    Monitor.Wait(_resultQueue);
                                if (_exception != null)
                                    throw _exception;
                                result = _resultQueue.Dequeue();
                            }
                            switch (result.ReturnStatus)
                            {
                                case FetchResult.Status.Failure:
                                case FetchResult.Status.Obsolete:
                                    continue;
                                case FetchResult.Status.Retry:
                                    lock (_codeQueue)
                                    {
                                        _codeQueue.Enqueue(result.Code);
                                    }
                                    continue;
                            }
                            UpdateDataFarm(result.Code, result.Prices);
                            SendMessage(AsyncConst.WM_ASYNCPROCESS, result.Code, AsyncConst.LPARAM_PROGRESS_SUCCESSFUL);
                        }
                        if (_codeQueue.Count == 0)
                            break;
                        if (retry++ == 10)
                            throw new Exception(string.Format("株価の取得に失敗しました。時間を置いて再試行してください。: {0}～{1}",
                                  _series[0], _series[_series.Count - 1]));
                        Thread.Sleep(10000);
                    }
                } while (dates.Count > 0);
            }
            finally
            {
                lock (_syncObject)
                {
                    _terminate = true;
                    Monitor.PulseAll(_syncObject);
                }
                foreach (var thread in threads)
                    thread.Join();
            }
        }

        public void UpdateDataFarm(int code, SortedDictionary<int, NewDailyData> prices)
        {
            var farm = (DailyDataFarm)Env.BrandCollection.FindBrand(code).CreateDailyFarm(prices.Count);
            var empty = farm.IsEmpty;
            var skip = true;
            foreach (var pair in prices)
            {
                if (empty && skip && pair.Value.volume == 0)
                    continue;
                skip = false;
                farm.UpdateDataFarm(pair.Key, pair.Value);
            }
            farm.Save(Util.GetDailyDataFileName(code));
        }

        private void RunFetchPrices()
        {
            var code = 0;
            try
            {
                while (true)
                {
                    lock (_syncObject)
                    {
                        while ((_codeQueue == null || _codeQueue.Count == 0) && !_terminate)
                            Monitor.Wait(_syncObject);
                        if (_terminate || _codeQueue == null)
                            return;
                        code = _codeQueue.Dequeue();
                    }
                    var result = FetchPrices(code, _series);
                    lock (_resultQueue)
                    {
                        _resultQueue.Enqueue(result);
                        Monitor.Pulse(_resultQueue);
                    }
                }
            }
            catch (Exception e)
            {
                lock (_resultQueue)
                {
                    _exception = new Exception(string.Format("{0}: {1} {2}", e.Message, code, _series[0]), e);
                    Monitor.Pulse(_resultQueue);
                }
            }
        }

        private FetchResult FetchPrices(int code, IList<int> dates)
        {
            string page;
            var status = GetPage(code, Util.IntToDate(dates[0]), Util.IntToDate(dates[dates.Count - 1]), out page);
            if (status == FetchResult.Status.Failure || status == FetchResult.Status.Retry)
                return new FetchResult { Code = code, ReturnStatus = status };
            return ParsePage(code, page, dates);
        }

        private FetchResult.Status GetPage(int code, DateTime begin, DateTime end, out string page)
        {
            if (code == (int)BuiltInIndex.Nikkei225)
                code = 998407;
            else if (code == (int)BuiltInIndex.TOPIX)
                code = 998405;
            var url = string.Format(
                "http://info.finance.yahoo.co.jp/history/?code={0}&sy={1}&sm={2}&sd={3}&ey={4}&em={5}&ed={6}&tm=d",
                code, begin.Year, begin.Month, begin.Day, end.Year, end.Month, end.Day);
            page = null;
            try
            {
                using (var reader = new StreamReader(Util.HttpDownload(url)))
                    page = reader.ReadToEnd();
            }
            catch (WebException e)
            {
                switch (e.Status)
                {
                    case WebExceptionStatus.ProtocolError:
                        switch (((HttpWebResponse)e.Response).StatusCode)
                        {
                            case (HttpStatusCode)999:
                            case HttpStatusCode.InternalServerError:
                            case HttpStatusCode.BadGateway:
                                return FetchResult.Status.Retry;
                        }
                        throw;
                    case WebExceptionStatus.Timeout:
                    case WebExceptionStatus.ConnectionClosed:
                    case WebExceptionStatus.ReceiveFailure:
                    case WebExceptionStatus.ConnectFailure:
                        return FetchResult.Status.Retry;
                    default:
                        throw;
                }
            }
            return FetchResult.Status.Success;
        }

        private FetchResult ParsePage(int code, string buf, IEnumerable<int> dates)
        {
            var valid = new Regex(
                @"<td>(?<year>\d{4})年(?<month>1?\d)月(?<day>\d?\d)日</td>" +
                "<td>(?<open>[0-9,.]+)</td><td>(?<high>[0-9,.]+)</td><td>(?<low>[0-9,.]+)</td>" +
                "<td>(?<close>[0-9,.]+)</td>(?:<td>(?<volume>[0-9,]+)</td>)?");
            var invalid = new Regex("該当する期間のデータはありません。<br>期間をご確認ください。");
            var obs = new Regex("該当する銘柄はありません。<br>再度銘柄（コード）を入力し、「表示」ボタンを押してください。");
            var empty = new Regex("<dl class=\"stocksInfo\">\n<dt></dt><dd class=\"category yjSb\"></dd>");

            if (buf == null)
                return null;
            var dict = new SortedDictionary<int, NewDailyData>();
            var matches = valid.Matches(buf);
            if (matches.Count == 0)
            {
                if (obs.Match(buf).Success || empty.Match(buf).Success) // 上場廃止(銘柄データが空のこともある)
                    return new FetchResult {ReturnStatus = FetchResult.Status.Obsolete};
                if (!invalid.Match(buf).Success)
                    throw new Exception("ページから株価を取得できません。");
                // ここに到達するのは出来高がないか株価が用意されていない場合
            }
            try
            {
                var shift = IsIndex(code) ? 100 : 10; // 指数は100倍、株式は10倍で記録する
                const NumberStyles s = NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands;
                foreach (Match m in matches)
                {
                    var date = new DateTime(int.Parse(m.Groups["year"].Value),
                        int.Parse(m.Groups["month"].Value),
                        int.Parse(m.Groups["day"].Value));
                    dict[Util.DateToInt(date)] = new NewDailyData
                    {
                        open = (int)(double.Parse(m.Groups["open"].Value, s) * shift),
                        high = (int)(double.Parse(m.Groups["high"].Value, s) * shift),
                        low = (int)(double.Parse(m.Groups["low"].Value, s) * shift),
                        close = (int)(double.Parse(m.Groups["close"].Value, s) * shift),
                        volume = m.Groups["volume"].Value == "" ? 0 : (int)double.Parse(m.Groups["volume"].Value, s)
                    };
                }
            }
            catch (FormatException e)
            {
                throw new Exception("ページから株価を取得できません。", e);
            }
            // 出来高がない日の株価データがないので値が0のデータを補う。
            foreach (var date in dates)
            {
                if (!dict.ContainsKey(date))
                    dict[date] = new NewDailyData();
            }
            return new FetchResult {Code = code, Prices = dict, ReturnStatus = FetchResult.Status.Success};
        }

        private bool IsIndex(int code)
        {
            return code == (int)BuiltInIndex.Nikkei225 ||
                   code == (int)BuiltInIndex.TOPIX;
        }
    }
}