OtoNoOto
2/28/2018 - 12:44 AM
CSV Importer that matches CSV column headers to a new object. This was developed by Adam and then retrofitted to SAIT's specific needs. I fi
CSV Importer that matches CSV column headers to a new object. This was developed by Adam and then retrofitted to SAIT's specific needs. I figure this is a good place to put this so we can collectively refine it.
App_Data
|__xml
|__Custom
|__CSVImport
|__Controllers
|__ImportController.cs
|__Models
|__CSVResourceMap.cs
|__DocumentImporter.cs
|__Response.cs
|__scripts
|__app.html
|__loglister.html
|__logloader.html
|__Views
|__Import
|__Index.cshtml<script type="text/javascript" src="../bower_scripts/dropzone/dist/min/dropzone.min.js"></script>
<link rel="stylesheet" href="../bower_scripts/dropzone/dist/min/dropzone.min.css" />
<link rel="import" href="../bower_scripts/iron-ajax/iron-ajax.html" />
<link rel="import" href="../bower_scripts/paper-card/paper-card.html">
<dom-module id='log-loader'>
<style>
paper-card {
margin-bottom: 16px;
width: 75%;
opacity: 0;
animation: fadein;
animation-fill-mode: forwards;
animation-duration: .3s;
text-align: left;
}
#logs {
height: 360px;
overflow: auto;
margin-top: 16px;
}
.cardRow {
text-align: center;
}
.card-content {
word-wrap: break-word;
}
@keyframes fadein {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
form{
text-align:center;
}
</style>
<template>
<form action="[[uploadurl]]" method="post" class="dropzone" id="csvdropzone"></form>
<div id="logs">
<div id="repeater">
<template is="dom-repeat" items="[[items]]">
<div class="cardRow">
<paper-card heading="[[item.Time]]">
<div class="card-content">[[item.Message]]</div>
</paper-card>
</div>
</template>
</div>
</div>
<iron-ajax id="ajax"
url="[[url]]"
body='{"offset":"[[offset]]"}'
handle-as="json"
content-type="application/json"
method="POST"
last-response="{{lastResponse}}"
on-response="processData"></iron-ajax>
<!-- scroll-target uses the document scroll -->
</template>
<script>
Polymer({
is: 'log-loader',
properties: {
offset: {
type: Number,
value: 0
},
items: {
type: Array,
value: []
},
lastResponse: {
type: Object,
value:{}
},
seen:{
type: Object,
value:{}
},
url:{
type:String,
value:""
},
uploadurl: {
type:String,
value:"/default"
}
},
timer:null,
ready: function () {
//this.$.scrollThreshold.scrollTarget = this.$.test;
var scope = this;
this.timer = setInterval(function () {
if (scope.$.ajax.activeRequests.length == 0) {
scope.$.ajax.generateRequest()
}
}, 1500);
if (window.igxDropzone) {
window.igxDropzone.url = this.uploadurl;
}
},
cleared: false,
clear: function () {
var scope = this;
scope.seen = {};
while (scope.items.length > 0) {
scope.pop('items');
}
scope.offset = 0;
scope.cleared = false;
window.igxDropzone.enable();
window.igxDropzone.removeAllFiles();
setTimeout(function () { }, 0);
},
unique : function(list){
var u = {}, a = [];
for (var i = 0, l = list.length; i < l; ++i) {
if (u.hasOwnProperty(list[i])) {
continue;
}
a.push(list[i]);
u[list[i]] = 1;
}
return a;
},
clearTimer:null,
processData: function (evt) {
var scope = this;
if (!evt.detail.response) {
return;
}
var status = evt.detail.response;
if (!scope.cleared && scope.items.length > 0 && status.currentTask == 0 && status.messages.length == 0) {
scope.cleared = true;
setTimeout(function () { scope.clear(); }, 7500);
}
for (var i in status.messages) {
var message = status.messages[i];
if (status.currentTask != 0) {
window.igxDropzone.disable();
}
var cleanTime = String(message.Time).replace(/^\/Date\(/g, '');
cleanTime = String(cleanTime).replace(/\)\/$/g, '');
message.TimeNumber = parseInt(cleanTime);
if (!scope.seen[message.TimeNumber]) {
if (this.clearTimer) {
clearInterval(this.clearTimer);
this.clear();
}
scope.offset++;
message.Time = new Date(parseInt(cleanTime)).toLocaleDateString('en-us');
scope.seen[message.TimeNumber] = true;
scope.unshift('items', message);
}
}
}
});
window.addEventListener('WebComponentsReady', function csvload(e) {
window.removeEventListener("WebComponentsReady", csvload, false);
var form = document.querySelector("#csvdropzone");
window.igxDropzone = new Dropzone(form, {
paramName: "file",
maxFilesize: 2000000,
clickable: true,
maxFiles: 1,
acceptedFiles: ".csv,.zip",
init: function () {
this.on("success", function () { this.disable() });
}
});
});
</script>
</dom-module><link rel="import" href="../bower_scripts/iron-ajax/iron-ajax.html" />
<link rel="import" href="../bower_scripts/paper-card/paper-card.html">
<link rel="import" href="../bower_scripts/iron-scroll-threshold/iron-scroll-threshold.html">
<dom-module id='log-lister'>
<style>
paper-card {
margin-bottom: 16px;
width: 75%;
opacity: 0;
animation: fadein;
animation-fill-mode: forwards;
animation-duration: .3s;
text-align: left;
}
#logslist {
height: 535px;
overflow: auto;
margin-top: 16px;
}
.card-content {
word-wrap: break-word;
}
.cardRow {
text-align: center;
}
@keyframes fadein {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
</style>
<template>
<iron-ajax id="ajaxlogs"
url="[[url]]"
body='{"offset":"[[offset]]"}'
handle-as="json"
method="POST"
debounce-duration="500"
last-response="{{lastResponse}}"
content-type="application/json"
on-response="processData"></iron-ajax>
<div id="logslist">
<div>
<template is="dom-repeat" items="[[items]]">
<div class="cardRow">
<paper-card>
<div class="card-content">[[item]]</div>
</paper-card>
</div>
</template>
</div>
</div>
<iron-scroll-threshold id="scrollThreshold" lower-triggered="{{lowerTriggered}}" lower-threshold="20" on-lower-trigger="loadMoreData" fit>
</iron-scroll-threshold>
</template>
<script>
Polymer({
is: 'log-lister',
properties: {
offset: {
type: Number,
value: 0
},
items: {
type: Array,
value: []
},
lastResponse: {
type: Object,
value:{}
},
url: {
type: String,
value: ""
}
},
timer:null,
ready: function () {
var scope = this;
this.$.scrollThreshold.scrollTarget = this.$.logslist;
this.timer = setInterval(function () {
if (scope.$.ajaxlogs.activeRequests.length == 0 && scope.items.length == 0) {
scope.loadMoreData();
}
}, 2500);
},
processData: function (evt) {
var scope = this;
if (!evt.detail.response) {
return;
}
var status = evt.detail.response;
for (var i in status) {
var message = status[i];
//if (scope.items.indexOf(message) == -1) {
scope.offset++;
scope.push('items', message);
//}
}
if (this.timer && this.items.length > 0) {
clearInterval(this.timer);
}
},
loadMoreData: function () {
var scope = this;
if (scope.$.ajaxlogs.activeRequests.length == 0) {
scope.$.ajaxlogs.generateRequest();
}
scope.$.scrollThreshold.clearLower();
}
});
</script>
</dom-module><link rel="import" href="../bower_scripts/iron-flex-layout/iron-flex-layout.html">
<link rel="import" href="../bower_scripts/iron-pages/iron-pages.html" />
<link rel="import" href="../bower_scripts/paper-tabs/paper-tabs.html" />
<link rel="import" href="../bower_scripts/app-layout/app-toolbar/app-toolbar.html">
<link rel="import" href="../bower_scripts/app-layout/app-scroll-effects/app-scroll-effects.html">
<link rel="import" href="../bower_scripts/app-layout/app-header/app-header.html">
<link rel="import" href="./loglister.html">
<link rel="import" href="./logloader.html">
<dom-module id='csv-app'>
<style>
app-header {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 132px;
color: #fff;
background-color: #4285f4;
}
app-toolbar {
margin-bottom: 20px;
}
.page-content {
padding-top: 132px;
}
log-loader, log-lister {
overflow: hidden;
}
</style>
<template>
<app-header condenses reveals fixed effects="waterfall">
<app-toolbar>
<h1 title>SAIT Course Catalogue Import</h1>
</app-toolbar>
<paper-tabs selected="{{tabDex}}">
<paper-tab>Overview</paper-tab>
<paper-tab>Upload Resource</paper-tab>
<paper-tab>Logs</paper-tab>
</paper-tabs>
</app-header>
<iron-pages class="page-content" selected="{{tabDex}}">
<div>
To upload an appropriately formatted CSV file, select the Upload Resources tab. Then drag and drop your CSV file into the drop area. The upload will begin immediately.
</div>
<div><log-loader uploadurl="[[uploadurl]]" url="[[statusurl]]"></log-loader></div>
<div><log-lister url="[[logurl]]"></log-lister></div>
</iron-pages>
</template>
<script>
Dropzone.autoDiscover = false;
Polymer({
is: 'csv-app',
properties: {
statusurl: {
type: String,
value: ""
},
logurl: {
type: String,
value: ""
},
uploadurl: {
type: String,
value: ""
},
tabDex: {
type: Number,
value: 0
}
},
ready: function () {
console.log(this);
}
});
</script>
</dom-module>
@{
Layout = null;
}
<!DOCTYPE html>
<html id="html">
<head>
<title>Index</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1, user-scalable=no">
<meta name="mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-capable" content="yes">
<script src="~/AppAsset/CSVImport/bower_scripts/webcomponentsjs/webcomponents-lite.min.js"></script>
@if (Model.IsDebug)
{
<link rel="import" href="~/AppAsset/CSVImport/scripts/app.html" />
}
else
{
<link rel="import" href="~/AppAsset/CSVImport/scripts/app.min.html" />
}
<style is="custom-style">
html{
overflow:hidden;
}
body {
margin: 0;
font-family: arial;
background-color: #eee;
}
</style>
</head>
<body unresolved>
<csv-app uploadurl="~/Apps/CSVImport/Import/Upload" logurl="~/Apps/CSVImport/Import/Logs" statusurl="~/Apps/CSVImport/Import/ImportStatus"></csv-app>
<script type="text/javascript">
window.addEventListener("dragenter", function (e) {
e.preventDefault();
e.dataTransfer.effectAllowed = "none";
e.dataTransfer.dropEffect = "none";
}, false);
window.addEventListener("dragover", function (e) {
e.preventDefault();
e.dataTransfer.effectAllowed = "none";
e.dataTransfer.dropEffect = "none";
});
window.addEventListener("drop", function (e) {
e.preventDefault();
e.dataTransfer.effectAllowed = "none";
e.dataTransfer.dropEffect = "none";
});
</script>
</body>
</html>
using CsvHelper;
using Ingeniux.CMS.Applications;
using Ingeniux.CMS.RavenDB.Indexes;
using Ionic.Zip;
using MoreLinq;
using NLog;
using Raven.Client.Linq;
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using System.Web.Mvc;
namespace Ingeniux.CMS.Controller.Custom
{
[Export(typeof(CMSControllerBase))]
[ExportMetadata("controller", "ImportController")]
[PartCreationPolicy(System.ComponentModel.Composition.CreationPolicy.NonShared)]
public class ImportController : CustomTabApplicationController
{
public const string RESOURCE_FOLDER = "~/App_Data/xml/Custom/CSVImport/App_Data/ResourceImport";
public const string LOGS_FILE = "~/App_Data/xml/Custom/CSVImport/App_Data/Logs.txt";
public const string STAGING_FOLDER = "~/App_Data/xml/Custom/CSVImport/App_Data/ResourceImport/Staging";
public const string COMP_SCHEMA_NAME = "CourseDetailPage";
public const string COMP_ROOT_XID = "x49";
public const string FOLDER_SCHEMA_NAME = "Folder";
private static List<Log> _MessageBox = new List<Log>();
private static Task _Task;
private static bool _Running = false;
private static ImporterTasks _CurrentTask = ImporterTasks.None;
private static string currentUser = "";
public string StagingPath
{
get
{
return Server.MapPath(STAGING_FOLDER);
}
}
public string LogPath
{
get
{
return Server.MapPath(LOGS_FILE);
}
}
public string ResourcePath
{
get
{
return Server.MapPath(RESOURCE_FOLDER);
}
}
public ActionResult Index()
{
bool isDebug = true;
#if DEBUG
isDebug = true;
#else
isDebug = false;
#endif
//System.Diagnostics.Debugger.Launch();
var model = new CSVModel();
model.IsDebug = isDebug;
return View(model);
}
static bool IsStalled { get; set; }
public JsonResult ImportStatus(int offset = 0)
{
if (_Task == null)
{
return Json(null);
}
if (_Task.IsCompleted && _CurrentTask != ImporterTasks.None)
{
if (_Task.IsFaulted)
{
_UpdateStatus(_Task.Exception.Message);
}
if (!IsStalled)
{
IsStalled = true;
_UpdateStatus("Process Stalled.");
}
this._CleanUp();
}
IEnumerable<Log> messages = _MessageBox.Skip(offset).Take(15);
var status = new StatusResponse();
status.complete = _Task != null ? _Task.IsCompleted : false;
status.messages = messages;
status.currentTask = _CurrentTask;
status.totalMessageCount = _MessageBox.Count();
return Json(status);
}
public JsonResult Logs(int offset = 0, int count = -1)
{
IEnumerable<string> lines = null;
bool hasLock = false;
Monitor.Enter(_CleanLock, ref hasLock);
if (hasLock)
{
if (!System.IO.File.Exists(LogPath))
{
System.IO.File.AppendAllLines(LogPath, new string[] { String.Empty });
}
if (_Task != null && _Task.IsFaulted && _CurrentTask != ImporterTasks.None)
{
_UpdateStatus(_Task.Exception.Message);
this._CleanUp();
}
lines = System.IO.File.ReadLines(LogPath).Skip(offset).Where(s => !String.IsNullOrEmpty(s)).Take(25);
Monitor.Exit(_CleanLock);
}
return Json(lines);
}
private void _UpdateStatus(string messageText)
{
bool hasLock = false;
Monitor.Enter(_CleanLock, ref hasLock);
if (hasLock)
{
try
{
var log = new Log();
log.Message = messageText;
log.Time = DateTime.Now;
messageText = log.Time.ToFullDateTimeString() + messageText + ": " + _Common.CurrentUser.UserId;
System.IO.File.AppendAllLines(LogPath, new string[] { messageText });
_Common.Log(LogLevel.Debug, messageText);
_MessageBox.Add(log);
}
catch (Exception e)
{
throw new Exception("Log Broke");
}
Monitor.Exit(_CleanLock);
}
}
private static Object _CleanLock = new Object();
private void _CleanUp()
{
bool hasLock = Monitor.TryEnter(_CleanLock);
if (hasLock)
{
_UpdateStatus("Update Complete.");
Task.Run(() =>
{
Thread.Sleep(8000);
IsStalled = false;
_MessageBox = new List<Log>();
_Running = false;
_CurrentTask = ImporterTasks.None;
Monitor.Exit(_CleanLock);
});
}
}
private void _UpdateStagingFolder()
{
_UpdateStatus("Updating Staging Folder");
DirectoryInfo stagingDir = new DirectoryInfo(StagingPath);
DocumentImporter docImporter = new DocumentImporter(_Common.ContentStore.XmlDirectoryPath);
if (!stagingDir.Exists)
{
try
{
stagingDir = Directory.CreateDirectory(StagingPath);
}
catch (AggregateException ae)
{
foreach (Exception e in ae.InnerExceptions)
{
_UpdateStatus(e.Message);
}
}
catch (Exception e)
{
_UpdateStatus(e.Message);
throw;
}
}
IEnumerable<FileInfo> stagedFiles = stagingDir.EnumerateFiles();
IEnumerable<string> fileNames = stagedFiles.Select(f => f.Name);
_UpdateStatus(fileNames.Count() + " Files Staged");
var nameBatch = fileNames.Batch(64);
List<Page> pages = new List<Page>();
int count;
using (IUserWriteSession session = this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser))
{
foreach (var batch in nameBatch)
{
IEnumerable<Page> pageBatch = (session.Site as Site).Query<Page, ISite, PagesByHierarchy>(
-1,
-1,
out count,
page => (page.SchemaName == COMP_SCHEMA_NAME && page._Name.In(batch) && !page.HierarchyValue.StartsWith(Site.RECYCLE_FOLDER_HIERARCHY))
);
pages.AddRange(pageBatch);
}
_UpdateStatus(pages.Count() + " Pages Found");
Dictionary<string, Page> pagesByName = pages.ToDictionaryDistinct(p => p.Name, p => p);
foreach (FileInfo staged in stagedFiles)
{
try
{
if (pagesByName.ContainsKey(staged.Name))
{
Page docPage = pagesByName[staged.Name];
IElement pathElement = docPage.Element("Path");
if (pathElement == null)
{
continue;
}
string pathValue = pathElement.Value;
if (String.IsNullOrEmpty(pathValue))
{
continue;
}
_UpdateStatus(staged.FullName + " Importing");
var file = docImporter.DocumentImport(staged.FullName, pathValue.Replace('/', '\\'));
if (file != null)
{
_UpdateStatus("Versioned File: " + file.FullName);
}
_UpdateStatus(staged.FullName + " Imported");
}
}
catch (Exception e)
{
_UpdateStatus(e.Message);
}
}
}
}
private void _ImportZip(string path)
{
_CurrentTask = ImporterTasks.Extracting;
try
{
_UpdateStatus("Importing Zip: " + path);
if (!Directory.Exists(StagingPath))
{
Directory.CreateDirectory(StagingPath);
}
using (ZipFile zipFile = ZipFile.Read(path))
{
foreach (ZipEntry e in zipFile)
{
e.Extract(StagingPath, ExtractExistingFileAction.OverwriteSilently);
}
//UpdateStagingFolder();
}
_UpdateStatus("Imported Zip Complete: " + path);
}
catch (Exception e)
{
_UpdateStatus(e.Message);
}
this._CleanUp();
}
private void _ImportCSV(string path)
{
_CurrentTask = ImporterTasks.ImportingDocuments;
try
{
_UpdateStatus("Importing CSV: " + path);
TextReader textReader = new StreamReader(path);
List<CSVPageWrapper> pageWrappers = null;
using (var csv = new CsvReader(textReader))
{
csv.Configuration.WillThrowOnMissingField = false;
csv.Configuration.RegisterClassMap<CSVResourceMap>();
pageWrappers = csv.GetRecords<CSVPageWrapper>().ToList();
}
_UpdateStatus(pageWrappers.Count() + " Records found");
if (pageWrappers == null)
{
return;
}
Dictionary<string, IPage> folderDict = new Dictionary<string, IPage>();
List<Page> pages = new List<Page>();
IEnumerable<string> names = pageWrappers.Select(w => w.Name);
var nameBatch = names.Batch(64);
Dictionary<string, Page> existingPages = new Dictionary<string, Page>();
using (IUserWriteSession session = this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser))
{
Schema schema = session.SchemasManager.SchemaByRootName(COMP_SCHEMA_NAME) as Schema;
Schema folderSchema = session.SchemasManager.SchemaByRootName(FOLDER_SCHEMA_NAME) as Schema;
if (schema == null)
{
string message = "No Schema with the root name: " + COMP_SCHEMA_NAME;
_UpdateStatus(message);
throw new Exception(message);
}
int count;
foreach (var batch in nameBatch)
{
IEnumerable<Page> pageBatch = (session.Site as Site).Query<Page, ISite, PagesByHierarchy>(
-1,
-1,
out count,
page =>
page.SchemaName == COMP_SCHEMA_NAME && page._Name.In(batch) &&
!page.HierarchyValue.StartsWith(Site.RECYCLE_FOLDER_HIERARCHY)
);
pages.AddRange(pageBatch);
}
existingPages = pages.ToDictionaryDistinct(p => p.Name, p => p);
}
var pageWrapperBatches = pageWrappers.Batch(32);
foreach (var pageWrapperBatch in pageWrapperBatches)
{
foreach (var wrapper in pageWrapperBatch)
{
using (
IUserWriteSession session =
this._Common.ContentStore.OpenWriteSession(this._Common.CurrentUser))
{
IPage rootFolder = session.Site.Page(COMP_ROOT_XID);
Schema schema = session.SchemasManager.SchemaByRootName(COMP_SCHEMA_NAME) as Schema;
Schema folderSchema = session.SchemasManager.SchemaByRootName(FOLDER_SCHEMA_NAME) as Schema;
if (existingPages.ContainsKey(wrapper.Name))
{
_UpdateStatus("Updating Page: " + wrapper.Name);
var page = session.Site.Page(existingPages[wrapper.Name].Id);
wrapper.UpdatePage(page);
_UpdateStatus("Updated Page: " + wrapper.Name);
}
else
{
rootFolder = session.Site.Page(COMP_ROOT_XID);
if (rootFolder == null)
{
string message = "No page with xID: " + COMP_ROOT_XID;
_UpdateStatus(message);
throw new Exception(message);
}
IPage currentFolder = rootFolder;
IEnumerable<string> folderPath = new[] { wrapper.Path };
string docPath = string.Empty;
foreach (string folderName in folderPath)
{
if (!String.IsNullOrEmpty(docPath))
{
docPath += "/";
}
docPath += folderName;
if (folderDict.ContainsKey(docPath))
{
currentFolder = folderDict[docPath];
}
else
{
//System.Diagnostics.Debugger.Launch();
IPage childFolder;
int count;
childFolder =
currentFolder.Children(out count)
.Where(p => p.Name == folderName)
.FirstOrDefault();
if (childFolder == null)
{
_UpdateStatus("Creating Folder: " + docPath);
currentFolder = session.Site.CreatePage(folderSchema, folderName,
currentFolder);
_UpdateStatus("Created Folder: " + docPath + currentFolder.Id);
}
else
{
currentFolder = childFolder;
}
folderDict[docPath] = currentFolder;
}
}
_UpdateStatus("Creating Component: " + docPath + wrapper.Name);
currentFolder = session.Site.Page(currentFolder.Id);
Page newPage = wrapper.CreatePage(currentFolder, schema) as Page;
_UpdateStatus("Created Component: " + docPath + newPage.Id);
if (newPage != null)
{
pages.Add(newPage);
}
}
}
}
}
//_UpdateStagingFolder();
_UpdateStatus("CSV Imported");
}
catch (AggregateException ae)
{
foreach (Exception e in ae.InnerExceptions)
{
_UpdateStatus(e.Message);
}
}
catch (Exception e)
{
_UpdateStatus(e.Message);
}
this._CleanUp();
}
public JsonResult Upload(HttpPostedFileBase file)
{
try
{
if (file == null)
{
this.Response.StatusCode = (int)HttpStatusCode.UnsupportedMediaType;
return Json(false);
}
ImportController._MessageBox = new List<Log>();
_UpdateStatus("File Uploaded: " + file.FileName);
Response r = new Response();
if ((_Task != null && !(_Task.IsCompleted || _Task.IsFaulted)) || _Running)
{
_UpdateStatus("File Uploaded But task in progress: " + file.FileName);
r.success = false;
r.message = "task in process";
return Json(r);
}
_Running = true;
if (file.ContentLength > 0)
{
var fileName = Path.GetFileName(file.FileName);
if (!Directory.Exists(ResourcePath))
{
_UpdateStatus("Creating Folder: " + ResourcePath);
Directory.CreateDirectory(ResourcePath);
}
var path = Path.Combine(ResourcePath, fileName);
_UpdateStatus("Saving File: " + path);
file.SaveAs(path);
_UpdateStatus("Saved File: " + path);
if (fileName.EndsWith(".zip"))
{
_Task = Task.Run(() => this._ImportZip(path));
}
else
{
_Task = Task.Run(() => this._ImportCSV(path));
}
}
r.success = true;
return Json(r);
}
catch (AggregateException ae)
{
foreach (Exception e in ae.InnerExceptions)
{
_UpdateStatus(e.Message);
}
Response r = new Response();
r.success = false;
r.message = "task in process";
return Json(r);
}
catch (Exception e)
{
_MessageBox = new List<Log>();
_Running = false;
_CurrentTask = ImporterTasks.None;
_UpdateStatus(e.Message);
Response r = new Response();
r.success = false;
r.message = "task in process";
return Json(r);
}
}
public JsonResult GetUploads()
{
return null;
}
public JsonResult LoadCSV(string FileName)
{
return null;
}
}
}using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Mvc;
namespace Ingeniux.CMS.Applications
{
public enum ImporterTasks
{
None,
Extracting,
ImportingDocuments,
UpdatingCompoonents
}
public class Response
{
public bool success;
public string message = String.Empty;
}
public class StatusResponse
{
public bool complete = true;
public ImporterTasks currentTask = ImporterTasks.None;
public IEnumerable<Log> messages;
public int totalMessageCount;
}
public class Log
{
public DateTime Time;
public string Message;
}
public class CSVModel
{
public bool IsDebug = false;
}
}using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using System.Web;
namespace Ingeniux.CMS.Applications
{
public class DocumentImporter
{
public static string SCHEMA_NAME = "Resource"; //move this to a config
public static string VERSION_DIRECTORY_NAME = "_versions";
private string _root;
public DocumentImporter(string xmlPath)
{
_root = xmlPath;
}
private void _VersionDocument(DirectoryInfo directoryInfo, string fileName)
{
string versionFolderPath = Path.Combine(directoryInfo.FullName,VERSION_DIRECTORY_NAME,fileName);
string fullFilePath = Path.Combine(directoryInfo.FullName,fileName);
DirectoryInfo versionDirInfo = Directory.CreateDirectory(versionFolderPath);
int versionNumber = 0;
if (versionDirInfo.GetFiles().Length > 0)
{
var regex = new Regex(@"^(\d+)\$");
var list = versionDirInfo.GetFiles("*" + fileName)
.Select(f =>
{
if (f.Name.Length <= 2)
{
return -1;
}
if (regex.IsMatch(f.Name))
{
return Int32.Parse(regex.Match(f.Name).Groups[1].Value);
}
return -1;
});
if(list.Count() > 0)
{
versionNumber = list.Max() + 1;
}
}
versionNumber = Math.Max(versionNumber, 1);
string versionFileName = String.Format(@"{0}${1}", versionNumber,fileName);
string fullVersionPath = Path.Combine(versionFolderPath, versionFileName);
System.IO.File.Move(fullFilePath, fullVersionPath);
}
public FileInfo DocumentImport(string url, string directoryPath, string fileName, bool createVersion = true)
{
string absDirectoryPath = Path.Combine(_root, directoryPath);
DirectoryInfo dirInfo = Directory.CreateDirectory(absDirectoryPath);
using (WebClient myWebClient = new WebClient())
{
// Download the Web resource and save it into the current filesystem folder.
string path = Path.Combine(dirInfo.FullName, fileName);
if (System.IO.File.Exists(path) && createVersion)
{
_VersionDocument(dirInfo, fileName);
}
try
{
myWebClient.DownloadFile(url, path);
}catch(Exception e)
{
return null;
}
return new FileInfo(path);
}
}
public FileInfo DocumentImport(string uriPath, string directoryPath, bool createVersion = true)
{
string absDirectoryPath = Path.Combine(_root, directoryPath);
DirectoryInfo dirInfo = Directory.CreateDirectory(absDirectoryPath);
FileInfo fileInfo = new FileInfo(uriPath);
if (fileInfo.Exists)
{
string path = Path.Combine(absDirectoryPath, fileInfo.Name);
if (System.IO.File.Exists(path) && createVersion)
{
_VersionDocument(dirInfo, fileInfo.Name);
}
fileInfo.MoveTo(path);
return fileInfo;
}
return null;
}
}
}using CsvHelper.Configuration;
using Ingeniux.CMS.Enums;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Ingeniux.CMS.Applications
{
public class CSVPageWrapper
{
private const string PAGE_ASSIGN_REASON = "Assigning for CSV update";
private const string OUTPUT_FOLDER = "Custom/";
private const string OUTPUT_EXTENSION = "igx";
private const string OUTPUT_DELIMITER = "~";
CSVPageWrapper()
{
}
public string Name { get; set; }
public string StartDate { get; set; }
public string EndDate { get; set; }
public char ConcatDelimiter
{
get;
set;
}
public string Path
{
get
{
int stopIndex = Math.Max(0, Elements["CourseCode"].IndexOfAny("0123456789".ToCharArray()));
return Elements["CourseCode"].Substring(0, stopIndex);
}
}
public Dictionary<string, string> Elements { get; set; }
public List<string> Categories { get; set; }
public IPage CreatePage(IPage parentPage, Schema schema)
{
try
{
IPage page = parentPage.Site.CreatePage(schema, Name, parentPage);
var firstOrDefault = page.AllElements().FirstOrDefault(e => e.Name == "SiteControl");
if (firstOrDefault != null)
{
firstOrDefault.Value = "x4";
}
this.UpdatePage(page);
return page;
}
catch (Exception e)
{
throw new Exception("Page Creation Error");
}
}
public void UpdatePage(IPage page)
{
string xmlPath = page.Site.ContentStore.XmlDirectoryPath;
if (page.CheckedOut)
{
page.CheckInSingleWithNoValidate(null);
}
page.CheckOut(false);
page.AssignUser(page.Site.CurrentUser, PAGE_ASSIGN_REASON);
//Setting a link on every element
var linkFirstOrDefault = page.AllElements().FirstOrDefault(e => e.Name == "RegisterLink") as LinkElement;
if (linkFirstOrDefault != null)
{
linkFirstOrDefault.URL =
"http://register.sait.ca/servlet/CourseController?calendarType=All&method=getCourseContent&courseCode=" +
this.Elements["UrlCode"];
linkFirstOrDefault.LinkType = EnumLinkElementType.IGX_EXTERNAL;
linkFirstOrDefault.SetAttributeValue("Target", "_blank");
linkFirstOrDefault.SetAttributeValue("Link", "Click to see course offerings.");
}
foreach (var ele in this.Elements)
{
var element = page.AllElements().FirstOrDefault(e => e.Name == ele.Key);
if (element != null)
{
var _value = ele.Value;
if (element.Type == Enums.EnumElementType.IGX_MULTI_SELECT || element.Type == Enums.EnumElementType.IGX_ENUMERATION)
{
string fileName = String.Format("{0}.{1}", element.Name, OUTPUT_EXTENSION);
string outputPath = System.IO.Path.Combine(xmlPath, OUTPUT_FOLDER, fileName);
string delimiter = string.Empty;
bool hasValue = false;
string output;
if (System.IO.File.Exists(outputPath))
{
IEnumerable<string> values = System.IO.File.ReadAllText(outputPath).Split('~');
if (!values.Contains(ele.Value))
{
delimiter = values.Count() > 0 ? OUTPUT_DELIMITER : string.Empty;
}
else
{
hasValue = true;
}
}
if (!hasValue)
{
output = String.Format("{0}{1}", delimiter, ele.Value);
System.IO.File.AppendAllText(outputPath, output);
}
}
if (element.Type == Enums.EnumElementType.IGX_DOCUMENT)
{
if (ele.Value.StartsWith("Documents/"))
{
_value = ele.Value.SubstringAfter("Documents/");
}
}
element.Value = _value;
}
}
foreach (var catName in this.Categories)
{
int count;
var cat = page.Site.Session.TaxonomyManager.Categories(out count, nameInitialLetters: catName).Where(c => c.Name == catName);
if (cat != null && cat.Count() > 0)
{
page.AddCategories(cat);
}
}
page.Name = Name;
try
{
page.StartDate = DateTime.Parse(StartDate);
}
catch (Exception e)
{
page.StartDate = null;
}
try
{
page.EndDate = DateTime.Parse(EndDate);
}
catch (Exception e)
{
page.EndDate = null;
}
}
}
public sealed class CSVResourceMap : CsvClassMap<CSVPageWrapper>
{
public string Name = "Name";
public string StartDate = "Available Date";
public string EndDate = "Expiration Date";
public string[][] colNames =
{
new string[]{"Title","Name"},
new string[]{"CourseCode","Prefix", "Code"},
new string[]{"CourseDescription","Course Description:"},
new string[]{"Credits","credits"},
new string[]{"Prereqs","Prerequisite(s): (Rendered)"},
new string[]{"Offered","Course Type"},
new string[] {"BrowserBarTitle", "Name"},
new string[] {"MetaDescription", "CourseDescription"},
new string[] {"UrlCode", "Prefix", "Code"},
};
public string[][] catNames =
{
//new string[]{"Regions"},
//new string[]{"Catalog Item Number"}
};
public Dictionary<string, string> ConcatDelimiter = new Dictionary<string, string>()
{
{"UrlCode", "-" }
};
public CSVResourceMap()
{
Map(m => m.Name).Name(Name);
Map(m => m.Elements).ConvertUsing(row =>
{
Dictionary<string, string> eles = new Dictionary<string, string>();
foreach (var cols in colNames)
{
string fieldValue;
eles[cols[0]] = String.Empty;
var delimit = ConcatDelimiter.Item(cols[0]) ?? "";
for (var i = 1; i < cols.Length; i++)
{
if (!row.TryGetField(cols[i], out fieldValue))
{
continue;
}
eles[cols[0]] = eles[cols[0]] + delimit + fieldValue;
}
eles[cols[0]] = eles[cols[0]].TrimStart(delimit.ToCharArray());
}
return eles;
});
Map(m => m.Categories).ConvertUsing(row =>
{
List<string> eles = new List<string>();
foreach (var cols in catNames)
{
var delimit = ConcatDelimiter.Item(cols[0]) ?? "";
string fieldValue;
string catName = String.Empty;
for (var i = 0; i < cols.Length; i++)
{
if (!row.TryGetField(cols[i], out fieldValue))
{
continue;
}
catName = catName + delimit + fieldValue;
}
eles.Add(catName.TrimStart(delimit.ToCharArray()));
}
return eles;
});
Map(m => m.StartDate).Name(StartDate);
Map(m => m.EndDate).Name(EndDate);
Map(m => m.ConcatDelimiter).Default(ConcatDelimiter);
}
}
}