Skip to content

V
Veget
Commit 79fa10af by pengzhigang
Options

迁移晶赞服务Pick My Sql 服务

parent 9d639cef
Showing with 196 additions and 96 deletions
.vs/VeGGet/v15/.suo
No preview for this file type
FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/Csv/CsvWriter.cs
......@@ -158,7 +158,39 @@ namespace ElencySolutions.CsvHelper
{
WriteCsv(dataTable, filePath, null);
}
public void WriteCsvHaveTitle(DataTable dataTable, string filePath)
{
WriteCsvHaveTitle(dataTable, filePath, null);
}
public void WriteCsvHvaeTitle(DataTable dataTable, string filePath, Encoding encoding)
{
if (File.Exists(filePath))
File.Delete(filePath);
FileStream _file = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite);
using (StreamWriter writer = new StreamWriter(_file, encoding ?? Encoding.Default, 1024 * 1024 * 1))
{
WriteToStream(dataTable, writer);
writer.Flush();
writer.Close();
_file.Close();
}
}
public void WriteCsvHaveTitle(DataTable dataTable, string filePath, Encoding encoding)
{
if (File.Exists(filePath))
File.Delete(filePath);
FileStream _file = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite);
using (StreamWriter writer = new StreamWriter(_file, encoding ?? Encoding.Default, 1024 * 1024 * 1))
{
WriteToStreamHaveTitle(dataTable, writer);
writer.Flush();
writer.Close();
_file.Close();
}
}
/// <summary>
/// Writes a DataTable to a file
/// </summary>
......@@ -300,8 +332,19 @@ namespace ElencySolutions.CsvHelper
WriteRecord(fields, writer);
}
}
private void WriteToStreamHaveTitle(DataTable dataTable, TextWriter writer)
{
List<string> headers = (from DataColumn column in dataTable.Columns select column.ColumnName).ToList();
WriteRecordCsv(headers, writer);
/// <summary>
foreach (DataRow row in dataTable.Rows)
{
var fields = row.ItemArray.ToList().ConvertAll(o => o.ToString());
WriteRecordCsv(fields, writer);
}
}
/// <summary>Csv
/// Writes the record to the underlying stream
/// </summary>
/// <param name="fields">Fields</param>
......@@ -341,6 +384,43 @@ namespace ElencySolutions.CsvHelper
writer.WriteLine();
}
//，号分割符
private void WriteRecordCsv(IList<string> fields, TextWriter writer)
{
for (int i = 0; i < fields.Count; i++)
{
bool quotesRequired = fields[i].Contains(",");
bool escapeQuotes = fields[i].Contains("\"");
string fieldValue = (escapeQuotes ? fields[i].Replace("\"", "\"\"") : fields[i]);
if (ReplaceCarriageReturnsAndLineFeedsFromFieldValues && (fieldValue.Contains("\r") || fieldValue.Contains("\n")))
{
quotesRequired = true;
fieldValue = fieldValue.Replace("\r\n", CarriageReturnAndLineFeedReplacement);
fieldValue = fieldValue.Replace("\r", CarriageReturnAndLineFeedReplacement);
fieldValue = fieldValue.Replace("\n", CarriageReturnAndLineFeedReplacement);
}
if (quotesRequired || escapeQuotes)
{
writer.Write("\"");
writer.Write(fieldValue);
writer.Write("\"");
}
else
{
writer.Write(fieldValue);
}
if (i < (fields.Count - 1))
{
writer.Write(",");
}
}
writer.WriteLine();
}
/// <summary>
/// Disposes of all unmanaged resources
/// </summary>
......
FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/Freemud.Quartz.MCDBigDataPickUp.csproj
......@@ -91,10 +91,10 @@
<Compile Include="Csv\CsvFile.cs" />
<Compile Include="Csv\CsvReader.cs" />
<Compile Include="Csv\CsvWriter.cs" />
<Compile Include="MCDBigDataPickUpUserPreOrderJob.cs" />
<Compile Include="MCDBigDataPickUpMiniProgramUserPreOrderJob.cs" />
<Compile Include="JobBase.cs" />
<Compile Include="Utility\MailHelper.cs" />
<Compile Include="Freemud.Quartz.MCDBigDataPickUp.cs" />
<Compile Include="MCDBigDataPickUpUserPreOrderJob.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Utility\FileHelper.cs" />
......
FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/Freemud.Quartz.MCDBigDataPickUp.cs → FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/MCDBigDataPickUpMiniProgramUserPreOrderJob.cs
......@@ -15,20 +15,34 @@ using ZhuiHu.Common;
namespace Freemud.Quartz.MCDBigDataPickUp
{
[DisallowConcurrentExecution]
public class MCDBigDataJob : JobBase
public class MCDBigDataPickUpMiniProgramUserPreOrderJob : JobBase
{
//麦当劳按三分钟一次，迁移后新代码，线上服务已替换为最新
public override void Execute(IJobExecutionContext context)
{
//晶赞迁移后新代码还未替换
//获取文件的名称
string tableFileName = context.MergedJobDataMap.GetString("tableFileName");
Logger.Info("start ----------------" + tableFileName);
string cnnString = context.MergedJobDataMap.GetString("connectionStr");
//读取时间文件的名称
string timeFileName = context.MergedJobDataMap.GetString("timeFileName");
string filePath = Path.Combine(Directory.GetCurrentDirectory(), timeFileName);
string sTime = FileHelper.ReadFile(filePath);
//当前前时间
string minutes = context.MergedJobDataMap.GetString("minutes");
//获取当前时间前五分钟
DateTime eTime = DateTime.Now.AddMinutes(-1 * int.Parse(minutes));
//获取table表时间列最大值
string sortName = context.MergedJobDataMap.GetString("sortName");
string TempSortName = context.MergedJobDataMap.GetString("TempSortName");
//获取目标盘
string aimPath= context.MergedJobDataMap.GetString("aimPath");
//设置报警邮件
string aimPath = context.MergedJobDataMap.GetString("aimPath");
//设置报警邮件
string smtp_account = context.MergedJobDataMap.GetString("smtp_account");
string smtp_psw = context.MergedJobDataMap.GetString("smtp_psw");
string alter_to = context.MergedJobDataMap.GetString("alter_to");
......@@ -38,29 +52,28 @@ namespace Freemud.Quartz.MCDBigDataPickUp
string smtp_port = context.MergedJobDataMap.GetString("smtp_port");
//数据库连接语句
string sql = context.MergedJobDataMap.GetString("sqlName");
Logger.Info("sql ----------------" + sql);
string commandText = context.MergedJobDataMap.GetString("commandText");
Logger.Info("sql ----------------" + commandText);
//加密密码
string password = context.MergedJobDataMap.GetString("password");
string days = context.MergedJobDataMap.GetString("days");
string predays = context.MergedJobDataMap.GetString("predays");
DateTime sTime = DateTime.Now.AddDays(-1 * int.Parse(predays));
DateTime eTime = DateTime.Now.AddDays(-1 * int.Parse(days));
Logger.Info("sTime eTime ----------------" + sTime+ eTime);
Logger.Info("sTime eTime ----------------" + sTime + eTime);
//DateTime fildNameDt = DateTime.Parse(eTime);
//文件上传SFTP IP,password
string sftpIP= context.MergedJobDataMap.GetString("sftpIP");
string sftpIP = context.MergedJobDataMap.GetString("sftpIP");
string sftpAccount = context.MergedJobDataMap.GetString("sftpAccount");
string sftpPassword = context.MergedJobDataMap.GetString("sftpPassword");
string sftpAmiPath = context.MergedJobDataMap.GetString("sftpAmiPath");
//文件保存路径
string fileName = sTime.ToString("yyyyMMddHH") + ".csv";
string fileNameZip = sTime.ToString("yyyyMMddHH") + ".zip";
string fileNameMd5 = sTime.ToString("yyyyMMddHH") + ".zip.md5";
string folderName = sTime.ToString("yyyyMMdd");
var strPath = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName+"\\"+fileName);
string fileName = eTime.ToString("yyyyMMddHHmmss") + ".csv";
string fileNameZip = eTime.ToString("yyyyMMddHHmmss") + ".zip";
string fileNameMd5 = eTime.ToString("yyyyMMddHHmmss") + ".zip.md5";
string folderName = eTime.ToString("yyyyMMdd");
var strPath = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileName);
var strPathZip = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileNameZip);
var strPathMd5 = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileNameMd5);
//生成文件夹的路径
......@@ -74,30 +87,37 @@ namespace Freemud.Quartz.MCDBigDataPickUp
// new SqlParameter("@EndTime", eTime.ToString()));
Logger.Info("strPathFolder ----------------" + tableFileName);
DataTable dt = Repository.excuteDataTable(sTime.ToString("yyyy-MM-dd"), eTime.ToString("yyyy-MM-dd"),sql);
DataTable dt = Repository.excuteDataTable(sTime.ToString(), eTime.ToString(), commandText);
Logger.Info("dt.Rows.Count ----------------" + tableFileName);
if (dt.Rows.Count == 0)
{
MailHelper.AlertEmailV1(alter_to, alter_to, alter_subject, alter_body, smtp_server, smtp_port, smtp_account, smtp_psw);
}
var dtTime = "";
if (dt.Rows.Count > 0)
{
if (!File.Exists(strPathFolder))
{
Directory.CreateDirectory(strPathFolder);
}
dtTime = dt.Compute("max(" + TempSortName + ")", "").ToString();
//删除临时添加的取时间的列
dt.Columns.Remove(TempSortName);
new CsvWriter().WriteCsv(dt, strPath);
FileProcessor.ProcessCSVFile(strPath, password);
string desPath = aimPath + "\\" + tableFileName + "\\" + folderName;
if (!File.Exists(desPath))
{
FileHelper.WriteFile(filePath, dtTime);
Directory.CreateDirectory(desPath);
}
if (!File.Exists(strPathFolder))
{
Directory.CreateDirectory(strPathFolder);
}
new CsvWriter().WriteCsv(dt, strPath);
FileProcessor.ProcessCSVFile(strPath, password);
string desPath = aimPath + "\\" + tableFileName + "\\" + folderName;
if (!File.Exists(desPath))
{
Directory.CreateDirectory(desPath);
}
//拷贝文件
System.IO.File.Copy(strPathMd5, Path.Combine(desPath, fileNameMd5), true);
System.IO.File.Copy(strPathZip, Path.Combine(desPath, fileNameZip), true);
......@@ -111,16 +131,16 @@ namespace Freemud.Quartz.MCDBigDataPickUp
sftp.Put(desPath + "\\" + fileNameMd5, sftpAmiPath + tableFileName, folderName, fileNameMd5);
sftp.Put(desPath + "\\" + fileNameZip, sftpAmiPath + tableFileName, folderName, fileNameZip);
}
//记录上次的时间戳以及记录log
Logger.Info(string.Format("startTime:{0} endTime:{1} dataCount:{2}, dataType:{3} ", sTime, eTime, dt.Rows.Count, tableFileName));
Logger.Info("end ----------------" + tableFileName);
//记录上次的时间戳以及记录log
Logger.Info(string.Format("startTime:{0} endTime:{1} dataCount:{2}, dataType:{3} ", sTime, eTime, dt.Rows.Count, tableFileName));
Logger.Info("end ----------------" + tableFileName);
}
catch (Exception ex)
{
Logger.Info("systerm error ----------------" + ex);
Logger.Error("PickupUserPreOrder数据生成失败,请检查服务" + DateTime.Now);
MailHelper.AlertEmailV1(alter_to, alter_to, alter_subject, alter_body, smtp_server, smtp_port, smtp_account, smtp_psw);
}
base.Execute(context);
......
FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/MCDBigDataPickUpUserPreOrderJob.cs
......@@ -16,33 +16,19 @@ namespace Freemud.Quartz.MCDBigDataPickUp
{
[DisallowConcurrentExecution]
public class MCDBigDataPickUpUserPreOrderJob : JobBase
{
//麦当劳按三分钟一次，迁移后新代码，线上服务已替换为最新
public override void Execute(IJobExecutionContext context)
{
//晶赞迁移后新代码还未替换
//获取文件的名称
string tableFileName = context.MergedJobDataMap.GetString("tableFileName");
Logger.Info("start ----------------" + tableFileName);
string cnnString = context.MergedJobDataMap.GetString("connectionStr");
//读取时间文件的名称
string timeFileName = context.MergedJobDataMap.GetString("timeFileName");
string filePath = Path.Combine(Directory.GetCurrentDirectory(), timeFileName);
string sTime = FileHelper.ReadFile(filePath);
//当前前时间
string minutes = context.MergedJobDataMap.GetString("minutes");
//获取当前时间前五分钟
DateTime eTime = DateTime.Now.AddMinutes(-1 * int.Parse(minutes));
//获取table表时间列最大值
string sortName = context.MergedJobDataMap.GetString("sortName");
string TempSortName = context.MergedJobDataMap.GetString("TempSortName");
//获取目标盘
string aimPath = context.MergedJobDataMap.GetString("aimPath");
//设置报警邮件
string aimPath= context.MergedJobDataMap.GetString("aimPath");
//设置报警邮件
string smtp_account = context.MergedJobDataMap.GetString("smtp_account");
string smtp_psw = context.MergedJobDataMap.GetString("smtp_psw");
string alter_to = context.MergedJobDataMap.GetString("alter_to");
......@@ -59,21 +45,22 @@ namespace Freemud.Quartz.MCDBigDataPickUp
string days = context.MergedJobDataMap.GetString("days");
string predays = context.MergedJobDataMap.GetString("predays");
Logger.Info("sTime eTime ----------------" + sTime + eTime);
DateTime sTime = DateTime.Now.AddDays(-1 * int.Parse(predays));
DateTime eTime = DateTime.Now.AddDays(-1 * int.Parse(days));
Logger.Info("sTime eTime ----------------" + sTime+ eTime);
//DateTime fildNameDt = DateTime.Parse(eTime);
//文件上传SFTP IP,password
string sftpIP = context.MergedJobDataMap.GetString("sftpIP");
string sftpIP= context.MergedJobDataMap.GetString("sftpIP");
string sftpAccount = context.MergedJobDataMap.GetString("sftpAccount");
string sftpPassword = context.MergedJobDataMap.GetString("sftpPassword");
string sftpAmiPath = context.MergedJobDataMap.GetString("sftpAmiPath");
//文件保存路径
string fileName = eTime.ToString("yyyyMMddHHmmss") + ".csv";
string fileNameZip = eTime.ToString("yyyyMMddHHmmss") + ".zip";
string fileNameMd5 = eTime.ToString("yyyyMMddHHmmss") + ".zip.md5";
string folderName = eTime.ToString("yyyyMMdd");
var strPath = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileName);
string fileName = sTime.ToString("yyyyMMddHH") + ".csv";
string fileNameZip = sTime.ToString("yyyyMMddHH") + ".zip";
string fileNameMd5 = sTime.ToString("yyyyMMddHH") + ".zip.md5";
string folderName = sTime.ToString("yyyyMMdd");
var strPath = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName+"\\"+fileName);
var strPathZip = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileNameZip);
var strPathMd5 = Path.Combine(Directory.GetCurrentDirectory(), "sftp" + "\\" + tableFileName + "\\" + folderName + "\\" + fileNameMd5);
//生成文件夹的路径
......@@ -87,37 +74,30 @@ namespace Freemud.Quartz.MCDBigDataPickUp
// new SqlParameter("@EndTime", eTime.ToString()));
Logger.Info("strPathFolder ----------------" + tableFileName);
DataTable dt = Repository.excuteDataTable(sTime.ToString(), eTime.ToString(), commandText);
DataTable dt = Repository.excuteDataTable(commandText);
Logger.Info("dt.Rows.Count ----------------" + tableFileName);
if (dt.Rows.Count == 0)
{
MailHelper.AlertEmailV1(alter_to, alter_to, alter_subject, alter_body, smtp_server, smtp_port, smtp_account, smtp_psw);
}
var dtTime = "";
if (dt.Rows.Count > 0)
{
if (!File.Exists(strPathFolder))
{
dtTime = dt.Compute("max(" + TempSortName + ")", "").ToString();
//删除临时添加的取时间的列
dt.Columns.Remove(TempSortName);
FileHelper.WriteFile(filePath, dtTime);
if (!File.Exists(strPathFolder))
{
Directory.CreateDirectory(strPathFolder);
}
new CsvWriter().WriteCsv(dt, strPath);
FileProcessor.ProcessCSVFile(strPath, password);
Directory.CreateDirectory(strPathFolder);
}
string desPath = aimPath + "\\" + tableFileName + "\\" + folderName;
if (!File.Exists(desPath))
{
new CsvWriter().WriteCsvHaveTitle(dt, strPath);
FileProcessor.ProcessCSVFile(strPath, password);
string desPath = aimPath + "\\" + tableFileName + "\\" + folderName;
if (!File.Exists(desPath))
{
Directory.CreateDirectory(desPath);
}
Directory.CreateDirectory(desPath);
}
//拷贝文件
System.IO.File.Copy(strPathMd5, Path.Combine(desPath, fileNameMd5), true);
System.IO.File.Copy(strPathZip, Path.Combine(desPath, fileNameZip), true);
......@@ -131,16 +111,16 @@ namespace Freemud.Quartz.MCDBigDataPickUp
sftp.Put(desPath + "\\" + fileNameMd5, sftpAmiPath + tableFileName, folderName, fileNameMd5);
sftp.Put(desPath + "\\" + fileNameZip, sftpAmiPath + tableFileName, folderName, fileNameZip);
}
//记录上次的时间戳以及记录log
Logger.Info(string.Format("startTime:{0} endTime:{1} dataCount:{2}, dataType:{3} ", sTime, eTime, dt.Rows.Count, tableFileName));
Logger.Info("end ----------------" + tableFileName);
//记录上次的时间戳以及记录log
Logger.Info(string.Format("startTime:{0} endTime:{1} dataCount:{2}, dataType:{3} ", sTime, eTime, dt.Rows.Count, tableFileName));
Logger.Info("end ----------------" + tableFileName);
}
catch (Exception ex)
{
Logger.Info("systerm error ----------------" + ex);
Logger.Error("PickupUserPreOrder数据生成失败,请检查服务" + DateTime.Now);
MailHelper.AlertEmailV1(alter_to, alter_to, alter_subject, alter_body, smtp_server, smtp_port, smtp_account, smtp_psw);
}
base.Execute(context);
......
FreeMud.Quartz/Freemud.Quartz.MCDBigDataPickUp/Utility/Repository.cs
......@@ -11,7 +11,22 @@ namespace Freemud.Quartz.MCDBigDataPickUp.Utility
{
public class Repository
{
public static DataTable excuteDataTable(string sTime, string eTime,string sql)
public static DataTable excuteDataTable(string sql)
{
try
{
using (DbManager db = ClusterConfig.CreateMySqlManager())
{
var ret = db.SetCommand(sql).ExecuteDataTable();
return ret;
}
}
catch (Exception ex)
{
throw ex;
}
}
public static DataTable excuteDataTable(string sTime ,string eTime,string sql)
{
try
{
......@@ -29,5 +44,6 @@ namespace Freemud.Quartz.MCDBigDataPickUp.Utility
}
}
}
}
VeGGet.sln

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 14
VisualStudioVersion = 14.0.25420.1
# Visual Studio 15
VisualStudioVersion = 15.0.28010.2046
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Lib", "Lib", "{9F05BAED-DEE0-48A3-94D9-B9AD81BA413D}"
ProjectSection(SolutionItems) = preProject
......@@ -111,10 +111,6 @@ EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Freemud.Quartz.IMAGPayData", "FreeMud.Quartz\Freemud.Quartz.IMAGPayData\Freemud.Quartz.IMAGPayData.csproj", "{BB99A190-6F03-4185-BDA0-FD72EAE4392C}"
EndProject
Global
GlobalSection(SubversionScc) = preSolution
Svn-Managed = True
Manager = AnkhSVN - Subversion Support for Visual Studio
EndGlobalSection
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|x86 = Debug|x86
......@@ -510,4 +506,11 @@ Global
{7071F204-38ED-4F09-8EAA-68412E7BCA79} = {69A3A4DD-75C5-40E9-BE9E-30143BB1E3C7}
{BB99A190-6F03-4185-BDA0-FD72EAE4392C} = {69A3A4DD-75C5-40E9-BE9E-30143BB1E3C7}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {CA41074F-6DE3-469F-8B8B-A0E49641CB76}
EndGlobalSection
GlobalSection(SubversionScc) = preSolution
Svn-Managed = True
Manager = AnkhSVN - Subversion Support for Visual Studio
EndGlobalSection
EndGlobal
ZhuiHu.Common/obj/Debug/ZhuiHu.Common.csproj.FileListAbsolute.txt
......@@ -32,3 +32,4 @@ D:\net\MaxPengTempFolder\CodeManeger\tempLatest\VeGGet\ZhuiHu.Common\obj\Debug\Z
D:\net\MaxPengTempFolder\CodeManeger\tempLatest\VeGGet\ZhuiHu.Common\obj\Debug\ZhuiHu.Common.csproj.CopyComplete
D:\net\MaxPengTempFolder\CodeManeger\tempLatest\VeGGet\ZhuiHu.Common\obj\Debug\ZhuiHu.Common.dll
D:\net\MaxPengTempFolder\CodeManeger\tempLatest\VeGGet\ZhuiHu.Common\obj\Debug\ZhuiHu.Common.pdb
D:\net\MaxPengTempFolder\CodeManeger\tempLatest\VeGGet\ZhuiHu.Common\obj\Debug\ZhuiHu.Common.csprojAssemblyReference.cache
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment