From cd0a9789ac6d8b4f6f978aed4effce63104fdc71 Mon Sep 17 00:00:00 2001 From: kyh <1329522@naver.com> Date: Wed, 17 Jan 2024 15:28:14 +0900 Subject: [PATCH] =?UTF-8?q?Saver=20=EA=B4=80=EB=A0=A8=20All=20Data=20?= =?UTF-8?q?=EC=A0=80=EC=9E=A5=20=EA=B8=B0=20=EB=8A=A5=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DataSaver/MainForm.cs | 64 ++++-- Million/Data/CandleData.cs | 6 +- Million/DbMangement/DBConnection.cs | 16 ++ Million/DbMangement/DBServiceFunctions.cs | 239 ++++++++++++++++++-- Million/WebMangement/WebServiceFunctions.cs | 2 + 5 files changed, 282 insertions(+), 45 deletions(-) diff --git a/DataSaver/MainForm.cs b/DataSaver/MainForm.cs index fc54b15..3aaacc1 100644 --- a/DataSaver/MainForm.cs +++ b/DataSaver/MainForm.cs @@ -21,7 +21,7 @@ namespace DataSaver Thread tickThread; System.Timers.Timer timer = null; string exchangedData = ""; - + public MainForm() { @@ -29,33 +29,33 @@ namespace DataSaver //CommonProperty.UpbitApi = new Million.WebMangement.Upbit.UpbitAPI(accessKey, secretKey); - InitForm(); + InitForm(); } /// /// 초기 동작 /// private void InitForm() - { + { // 거래소 정보 콤보박스 초기화 comboBoxEdit_ExchangePlace.Properties.Items.Clear(); // Million으로부터 거래소 정보 가져오기 - + comboBoxEdit_ExchangePlace.Properties.Items.AddRange(InitManager.GetMarketList()); - + // 종목 타입 설정 comboBoxEdit_PaymentType.Properties.Items.AddRange(InitManager.GetPaymentList()); comboBoxEdit_PaymentType.SelectedIndex = 0; /////TODO 임시로 Upbit 우선순위 설정 - comboBoxEdit_ExchangePlace.SelectedIndex = 0; + comboBoxEdit_ExchangePlace.SelectedIndex = 0; InitManager.InitFunc(comboBoxEdit_ExchangePlace.Text, Million.CommonProperty.PaymentType.KRW); for (int i = 0; i < checkedListBox1.Items.Count; i++) { checkedListBox1.SetItemChecked(i, true); - } + } } /// @@ -73,12 +73,12 @@ namespace DataSaver InitManager.UpdateMarketTableOfPlace(placeType, paymentType); //초창기 생성시에 데이터가 없음으로 Update이 후 재 처리 - if(CommonProperty.selectedMarketOfPlaceList == null || CommonProperty.selectedMarketOfPlaceList.Count == 0) + if (CommonProperty.selectedMarketOfPlaceList == null || CommonProperty.selectedMarketOfPlaceList.Count == 0) Million.DbManagement.DbServiceFunctions.GetMarketTable_FromDB(comboBoxEdit_ExchangePlace.SelectedItem.ToString()); // 데이터 입력 var marketList = CommonProperty.selectedMarketOfPlaceList; - comboBoxEdit_MarketInfo.Properties.Items.AddRange(marketList.Select(t => t.CoinName).ToList()); + comboBoxEdit_MarketInfo.Properties.Items.AddRange(marketList.Select(t => t.CoinName).ToList()); } /// @@ -101,7 +101,7 @@ namespace DataSaver private void button_SelectedMarketDataAcc_Click(object sender, EventArgs e) { //Create - if(!string.IsNullOrWhiteSpace(comboBoxEdit_PaymentType.SelectedItem?.ToString()) && + if (!string.IsNullOrWhiteSpace(comboBoxEdit_PaymentType.SelectedItem?.ToString()) && !string.IsNullOrWhiteSpace(comboBoxEdit_MarketInfo.SelectedItem?.ToString()) ) { @@ -137,6 +137,7 @@ namespace DataSaver list.Add((int)data); } + // 거래량 제외 Hunter foreach (var item in list) { var recentlyDbDataList = Million.DbManagement.DbServiceFunctions.GetOldNewDataList_FromDB @@ -144,6 +145,17 @@ namespace DataSaver Million.DbManagement.DbServiceFunctions.SetCandleData(placeName, type, recentlyDbDataList, (CommonProperty.MCandleTimeType)item); } + + //거래량 포함 Hunter + + foreach (var item in list) + { + var recentlyDbDataList = Million.DbManagement.DbServiceFunctions.GetOldNewDataList_FromDB_All + (marketList, placeName, (CommonProperty.MCandleTimeType)item); + + Million.DbManagement.DbServiceFunctions.SetCandleData_All(placeName, type, recentlyDbDataList, (CommonProperty.MCandleTimeType)item); + } + MessageBox.Show("Completed Set Candle Data"); } @@ -162,6 +174,8 @@ namespace DataSaver foreach (var item in marketNameList) sqlQuery += Million.DbManagement.DbServiceFunctions.CreateMarketTable_ToDB(item); + sqlQuery += Million.DbManagement.DbServiceFunctions.CreateMarketTable_ToDB_All(); + if (!string.IsNullOrWhiteSpace(sqlQuery)) Million.DbManagement.DBConnection.QueryUpdateExecuter(CommonProperty.DbConnectionStr + placeName, sqlQuery); @@ -170,7 +184,7 @@ namespace DataSaver /// /// 선택한 거래소의 종목(Market) 업데이트 /// - + /// /// Delete 기능 추가 /// @@ -181,7 +195,7 @@ namespace DataSaver var placeName = comboBoxEdit_ExchangePlace.SelectedItem.ToString(); CreateTableOfMarket(placeName); Million.DbManagement.DbServiceFunctions.GetMarketTable_FromDB(placeName); - + var dbMarketDataList = CommonProperty.selectedMarketOfPlaceList; if (string.IsNullOrWhiteSpace(de_start.Text)) @@ -193,11 +207,11 @@ namespace DataSaver var startDateTime = Convert.ToDateTime(de_start.Text).ToString("yyyy-MM-dd 00:00:00"); var endDateTime = Convert.ToDateTime(de_end.Text).ToString("yyyy-MM-dd 00:00:00"); - + var sqlQuery = ""; var count = 0; foreach (var item in dbMarketDataList) - { + { sqlQuery += string.Format("DELETE FROM [{0}].[dbo].[{1}] WHERE [DateTime] >= '{2}' AND [DateTime] <= '{3}';", placeName, item.CoinFullName, startDateTime, endDateTime); count++; if (count % 50 == 0) @@ -254,7 +268,7 @@ namespace DataSaver //var date = DateTime.Now; //var hour = date.Hour; - + var list = new List>(); var saveFlag = true; while (flag) @@ -265,7 +279,7 @@ namespace DataSaver foreach (var item in coinFullNameList) { - var dic = new Dictionary(); + var dic = new Dictionary(); var ticker = tickerList.FirstOrDefault(t => t.CoinFullName == item); dic["CoinFullName"] = item; dic["EndValue"] = ticker.TradePrice; @@ -273,27 +287,27 @@ namespace DataSaver list.Add(dic); } - if(date.Minute % 10 == 0 && saveFlag) + if (date.Minute % 10 == 0 && saveFlag) { list = list.OrderBy(t => ((DateTime)t["DateTime"])).ToList(); - var beforeDate = date.AddMinutes(-10); + var beforeDate = date.AddMinutes(-10); beforeDate = beforeDate.AddSeconds(-1 * beforeDate.Second); beforeDate = beforeDate.AddMilliseconds(-1 * beforeDate.Millisecond); var recentlyDate = date.AddSeconds(-1 * date.Second); recentlyDate = recentlyDate.AddMilliseconds(-1 * date.Millisecond); - + var dicList = list.Where(t => ((DateTime)t["DateTime"]) >= beforeDate && ((DateTime)t["DateTime"]) < recentlyDate).ToList(); DBConnection.QueryInsertExecuter_VerifyTickerData(dicList, beforeDate); - foreach(var item in dicList) + foreach (var item in dicList) { list.Remove(item); - } + } saveFlag = false; } - else if(date.Minute % 10 != 0) + else if (date.Minute % 10 != 0) { saveFlag = true; } @@ -311,12 +325,12 @@ namespace DataSaver // //var index = list.FindLastIndex(t => ((DateTime)t["DateTime"]).Hour == hour); // //list.RemoveRange(0, index + 1); - + // //date = DateTime.Now; // //hour = date.Hour; //} - Thread.Sleep(1000); + Thread.Sleep(1000); } } @@ -383,7 +397,7 @@ namespace DataSaver /// private void ScheduleFun(object sender, System.Timers.ElapsedEventArgs e) { - if(hours != 24) + if (hours != 24) { hours = 24; timer.Interval = hours * 60 * 60 * 1000; diff --git a/Million/Data/CandleData.cs b/Million/Data/CandleData.cs index 6365a0e..186d4a8 100644 --- a/Million/Data/CandleData.cs +++ b/Million/Data/CandleData.cs @@ -20,6 +20,9 @@ namespace Million.Data public double LowValue { get; set; } // 거래량 public double MoneyVolume { get; set; } + // 거래 대금 + public double TransactionAmount { get; set; } + public double BuyValue { get; set; } /// @@ -42,7 +45,8 @@ namespace Million.Data public Dictionary ParentCandleData = new Dictionary(); - public CandleType CandleType { get; set; } = CandleType.None; + public CandleType CandleType { get; set; } = CandleType.None; + /// /// 빗썸 DateTime 변환 로직 diff --git a/Million/DbMangement/DBConnection.cs b/Million/DbMangement/DBConnection.cs index 74285a9..9ca5781 100644 --- a/Million/DbMangement/DBConnection.cs +++ b/Million/DbMangement/DBConnection.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.IO; +using System.Linq; using System.Net.Sockets; using System.Runtime.Serialization.Formatters.Binary; using System.Text; @@ -115,6 +116,13 @@ namespace Million.DbManagement using (var rdr = cmd.ExecuteReader()) { + var columns = new List(); + + for (int i = 0; i < rdr.FieldCount; i++) + { + columns.Add(rdr.GetName(i)); + } + while (rdr.Read()) { var dateTime = rdr["DateTime"]?.ToString(); @@ -125,6 +133,8 @@ namespace Million.DbManagement double.TryParse(rdr["MoneyVolume"]?.ToString(), out var moneyVolume); int.TryParse(rdr["CandleTimeType"]?.ToString(), out var candleTimeType); + + var data = new CandleData(); data.DateTime = Convert.ToDateTime(dateTime); data.StartValue = startValue; @@ -134,6 +144,12 @@ namespace Million.DbManagement data.MoneyVolume = moneyVolume; data.CandleTimeType = candleTimeType; + if (columns.Any(t => t == "TransactionAmount")) + { + double.TryParse(rdr["TransactionAmount"]?.ToString(), out var transactionAmount); + data.TransactionAmount = transactionAmount; + } + returnData.Add(data); } } diff --git a/Million/DbMangement/DBServiceFunctions.cs b/Million/DbMangement/DBServiceFunctions.cs index a7ed95e..8a43d2b 100644 --- a/Million/DbMangement/DBServiceFunctions.cs +++ b/Million/DbMangement/DBServiceFunctions.cs @@ -8,6 +8,7 @@ using System.Windows.Forms; using System.Threading.Tasks; using Million.Manager; using Million.SaveData; +using DevExpress.Xpo.DB.Helpers; namespace Million.DbManagement { @@ -147,6 +148,23 @@ namespace Million.DbManagement return resultstring; } + public static string CreateMarketTable_ToDB_All() + { + var tableName = "AllDataTable"; + + return @"if not exists (select * from sysobjects where name='" + tableName + @"' and xtype='U') + create table " + "\"" + tableName + "\"" + @"( + CoinFullName nvarchar(50) not null, + DateTime datetime not null, + StartValue float not null, + EndValue float not null, + HighValue float not null, + LowValue float not null, + MoneyVolume float not null, + TransactionAmount float not null, + CandleTimeType int not null + ); "; + } /// /// DB 거래소 내 Market 테이블 생성 /// @@ -192,7 +210,7 @@ namespace Million.DbManagement var strDbCon = $"{CommonProperty.DbConnectionStr}Market"; var subSql = ""; - if(payment != null) + if (payment != null) { subSql = string.Format("AND [Payment] = '{0}'", payment); } @@ -201,7 +219,7 @@ namespace Million.DbManagement var sqlQuery = string.Format("select * from dbo.{0} where IsValid = 1 {1};", tableName, subSql); return DBConnection.QueryGetExecuter_Market(strDbCon, sqlQuery); - } + } /// /// 시간대 별 Data가져오는 함수 @@ -213,23 +231,23 @@ namespace Million.DbManagement public static CandleDataList GetCandleData(MarketData marketData, DateTime startDate, DateTime endDate, MCandleTimeType candleTimeType = MCandleTimeType.M1) { var returnData = new CandleDataList(); - if(marketData == null) + if (marketData == null) { return returnData; } var strDbCon = CommonProperty.DbConnectionStr + marketData.ExchangePlace; - if (candleTimeType == MCandleTimeType.M1 || candleTimeType == MCandleTimeType.M3 || candleTimeType == MCandleTimeType.M5 || - candleTimeType == MCandleTimeType.M10 | candleTimeType == MCandleTimeType.M15|| candleTimeType == MCandleTimeType.M30) + if (candleTimeType == MCandleTimeType.M1 || candleTimeType == MCandleTimeType.M3 || candleTimeType == MCandleTimeType.M5 || + candleTimeType == MCandleTimeType.M10 | candleTimeType == MCandleTimeType.M15 || candleTimeType == MCandleTimeType.M30) startDate = startDate.AddMinutes(-80 * (int)candleTimeType); - else if(candleTimeType == MCandleTimeType.H1 | candleTimeType == MCandleTimeType.H4) + else if (candleTimeType == MCandleTimeType.H1 | candleTimeType == MCandleTimeType.H4) { var val = 1; if (candleTimeType == MCandleTimeType.H4) val = 4; startDate = startDate.AddHours(-80 * val); - } - else if(candleTimeType == MCandleTimeType.D1) + } + else if (candleTimeType == MCandleTimeType.D1) startDate = startDate.AddDays(-80 * 1); else if (candleTimeType == MCandleTimeType.W1) startDate = startDate.AddDays(-80 * 7); @@ -262,7 +280,7 @@ namespace Million.DbManagement { return returnData; } - var strDbCon = CommonProperty.DbConnectionStr + marketData.ExchangePlace; + var strDbCon = CommonProperty.DbConnectionStr + marketData.ExchangePlace; if (candleTimeType == MCandleTimeType.M1 || candleTimeType == MCandleTimeType.M3 || candleTimeType == MCandleTimeType.M5 || candleTimeType == MCandleTimeType.M10 | candleTimeType == MCandleTimeType.M15 || candleTimeType == MCandleTimeType.M30) @@ -293,7 +311,59 @@ namespace Million.DbManagement return returnData; } - + /// + /// 전체 데이터를 하나의 테이블에서 검색(Old, New) + /// + /// + /// + /// + /// + public static List GetOldNewDataList_FromDB_All(List marketNameList, string databaseName, MCandleTimeType candleTimeType = MCandleTimeType.M1) + + { + var returnData = new List(); + var strDbCon = CommonProperty.DbConnectionStr + databaseName; + + //최신 데이터 + foreach (var item in marketNameList) + { + var data = new CandleDataList(); + data.ConGoodsData = item; + + var sqlQuery = string.Format(@"SELECT TOP(1) * FROM [AllDataTable] WHERE [CandleTimeType] = {0} ORDER BY [DateTime] DESC", ((int)candleTimeType).ToString()); + var recentlyDataList = DBConnection.QueryGetExecuter_Candle(strDbCon, sqlQuery); + + if (recentlyDataList.Count > 0) + { + recentlyDataList[0].CandleType = CandleType.New; + data.CandleDataLists_Im = recentlyDataList; + } + else + { + data.CandleDataLists_Im = new List(); + } + returnData.Add(data); + } + + //오래된 데이터 + foreach (var item in returnData) + { + if (item.CandleDataLists_Im.Count == 0) + continue; + + // 1개라도 있을 경우 + var sqlQuery = string.Format("SELECT TOP(1) * FROM [AllDataTable] WHERE [CandleTimeType] = {0} ORDER BY [DateTime] ASC", ((int)candleTimeType).ToString()); + + var recentlyDataList = DBConnection.QueryGetExecuter_Candle(strDbCon, sqlQuery); + if (recentlyDataList.Count > 0) + { + recentlyDataList[0].CandleType = CandleType.Old; + item.CandleDataLists_Im.AddRange(recentlyDataList); + } + } + + return returnData; + } /// /// DB에서 각 Market 별 가장 최신, 오래된 데이터 한개씩 가져오기 @@ -305,7 +375,7 @@ namespace Million.DbManagement { var returnData = new List(); - var strDbCon = CommonProperty.DbConnectionStr + databaseName; + var strDbCon = CommonProperty.DbConnectionStr + databaseName; //최신 데이터 foreach (var item in marketNameList) @@ -313,7 +383,7 @@ namespace Million.DbManagement var data = new CandleDataList(); data.ConGoodsData = item; - var sqlQuery = string.Format(@"SELECT TOP(1) [DateTime],[StartValue],[EndValue],[HighValue],[LowValue],[MoneyVolume],[CandleTimeType] FROM [{0}] WHERE [CandleTimeType] = {1} ORDER BY [DateTime] DESC", item.CoinFullName, ((int) candleTimeType).ToString()); + var sqlQuery = string.Format(@"SELECT TOP(1) [DateTime],[StartValue],[EndValue],[HighValue],[LowValue],[MoneyVolume],[CandleTimeType] FROM [{0}] WHERE [CandleTimeType] = {1} ORDER BY [DateTime] DESC", item.CoinFullName, ((int)candleTimeType).ToString()); var recentlyDataList = DBConnection.QueryGetExecuter_Candle(strDbCon, sqlQuery); if (recentlyDataList.Count > 0) @@ -411,6 +481,134 @@ namespace Million.DbManagement // } //} + /// + /// All Data에 Insert + /// + /// + /// + /// + /// + /// + public static bool SetCandleData_All(string placeName, ExchangeWebReqType place, List recentlyDbDataList, MCandleTimeType candleTimeType = MCandleTimeType.M1) + { + bool flag = false; + + //데이터 쌓는 부분은 재귀로 처리할 예정 + /** + * 단계 조정 + * 아무것도 없을 경우 0 + * 있을 경우1 + * 완료 2 + */ + + var insertQuery = ""; + + foreach (var item in recentlyDbDataList) + { + var sw = new System.Diagnostics.Stopwatch(); + sw.Start(); + var dateTime = DateTime.Now; + var tableName = item.ConGoodsData.CoinFullName; + int step = 0; + bool firstFlag = true; + var recentlyFlag = true; + switch (place) + { + case ExchangeWebReqType.UpbitCandleStick: + + //최신 부터 + var recentlyDateTime = new DateTime(1970, 1, 1); + var oldDateTime = new DateTime(1970, 1, 1); + CandleData recentlyData = null; + CandleData oldData = null; + + if (item.CandleDataLists_Im.Count > 0) + { + recentlyData = item.CandleDataLists_Im.Find(t => t.CandleType == CandleType.New); + recentlyDateTime = recentlyData.DateTime; + oldData = item.CandleDataLists_Im.Find(t => t.CandleType == CandleType.Old); + oldDateTime = oldData.DateTime; + + step = 1; + } + + while (true) + { + var webDataList = UpbitApi.GetCandleStics(item.ConGoodsData.CoinFullName, candleTimeType, dateTime); + + var insertDataList = new List(); + + if (step == 0) + { + insertDataList = webDataList; + } + else if (step == 1) + { + insertDataList.AddRange(webDataList.Where(t => t.DateTime > recentlyDateTime).ToList()); + insertDataList.AddRange(webDataList.Where(t => t.DateTime < oldDateTime).ToList()); + } + + insertDataList = insertDataList.OrderBy(t => t.DateTime).ToList(); + + if (firstFlag) + { + if (insertDataList.Count > 0) + { + insertDataList.Remove(insertDataList[insertDataList.Count - 1]); + firstFlag = false; + } + else + { + flag = true; + } + } + + //데이터 insert + foreach (var data in insertDataList) + { + insertQuery += string.Format("INSERT INTO [dbo].[AllDataTable]([CoinFullName],[DateTime],[StartValue], [EndValue], [HighValue], [LowValue], [MoneyVolume], [CandleTimeType], [TransactionAmount]) VALUES ('{1}',{2},{3},{4},{5},{6},{7},{8})", + tableName, data.DateTime.ToString("yyyy-MM-dd HH:mm:ss"), data.StartValue, data.EndValue, data.HighValue, data.LowValue, data.MoneyVolume, data.CandleTimeType, data.TransactionAmount); + } + + if (insertDataList.Count == 0) + { + if (recentlyFlag) + { + recentlyFlag = false; + dateTime = oldDateTime; + } + else + { + step = 2; + } + } + else + { + dateTime = insertDataList[0].DateTime; + } + + if (step == 2) + break; + //초당 8회 이상안됨 (0.125초) + if (!string.IsNullOrWhiteSpace(insertQuery)) + DBConnection.QueryUpdateExecuter(DbConnectionStr + placeName, insertQuery); + Thread.Sleep(60); + insertQuery = ""; + } + + break; + case ExchangeWebReqType.BithumbCandleTicker: + break; + + } + + sw.Stop(); + Console.WriteLine("Completed Set Candle Data of Table : " + tableName + ", Times : " + sw.Elapsed.ToString()); + } + return flag; + } + + /// /// 최신값, 가장오래된 값을 기준으로 Data 호출 및 처리 /// @@ -462,7 +660,7 @@ namespace Million.DbManagement while (true) { var webDataList = UpbitApi.GetCandleStics(item.ConGoodsData.CoinFullName, candleTimeType, dateTime); - + var insertDataList = new List(); if (step == 0) @@ -477,12 +675,15 @@ namespace Million.DbManagement insertDataList = insertDataList.OrderBy(t => t.DateTime).ToList(); - if (firstFlag) { - if(insertDataList.Count > 0) { + if (firstFlag) + { + if (insertDataList.Count > 0) + { insertDataList.Remove(insertDataList[insertDataList.Count - 1]); firstFlag = false; } - else { + else + { flag = true; } } @@ -507,7 +708,7 @@ namespace Million.DbManagement } } else - { + { dateTime = insertDataList[0].DateTime; } @@ -523,7 +724,7 @@ namespace Million.DbManagement break; case ExchangeWebReqType.BithumbCandleTicker: break; - + } sw.Stop(); @@ -538,7 +739,7 @@ namespace Million.DbManagement /// public static string InsertRealTimeIndicatorQuery(SavedResultData item, string name) { - + var str = string.Format(@"INSERT INTO [ResultData].[dbo].[RealTimeIndicator] ( [Name] diff --git a/Million/WebMangement/WebServiceFunctions.cs b/Million/WebMangement/WebServiceFunctions.cs index 5294a60..f47addf 100644 --- a/Million/WebMangement/WebServiceFunctions.cs +++ b/Million/WebMangement/WebServiceFunctions.cs @@ -593,11 +593,13 @@ namespace Million.WebServices double.TryParse(item["low_price"]?.ToString(), out var lowValue); //이거는 확인 필요 double.TryParse(item["candle_acc_trade_volume"]?.ToString(), out var moneyVolume); + double.TryParse(item["candle_acc_trade_price"]?.ToString(), out var transactionAmount); candleList.Add(new CandleData() { DateTime = Convert.ToDateTime(item["candle_date_time_kst"]?.ToString()), StartValue = startValue, + TransactionAmount = transactionAmount, EndValue = endValue, HighValue = highValue, LowValue = lowValue,