Dictionary ex

   IDictionary<int, byte[]> meDictionary = new Dictionary<int, byte[]>();
            for (int i = 0; i < 1100; i++)
            {
                var memoryPressure = new byte[1024 * 1024 * 1];
                meDictionary.Add(i, memoryPressure);
            }
            var memoryUsageAtEnd = GC.GetTotalMemory(false);

            //Update
            foreach (KeyValuePair<int, byte[]> pair in meDictionary)
            {
                pair.Value = "";
            }

TPL Demo for BEAM

Focus Areas

Try to improve current multi threading program with having one load but running
Linear Processing,
Thread,QueueWorkUserItem, and
Parallel Processing in the same instance

1) Difference between a Thread & a Task
Illustrate with an example
http://www.codeproject.com/Articles/152765/Task-Parallel-Library-of-n

2) how to read of Task Values.
Compare it with Thread.QueueWorkUserItem or Async Delegates
http://www.codeproject.com/Articles/152765/Task-Parallel-Library-of-n

3) Thread Continuation
http://www.codeproject.com/Articles/159533/Task-Parallel-Library-of-n

4) how to cancel Tasks
http://www.codeproject.com/Articles/152765/Task-Parallel-Library-of-n

Illustrate with a Chained one.

5) How to handle Exceptions

using Try catch block
http://www.codeproject.com/Articles/152765/Task-Parallel-Library-of-n

6) Data Parallelism
Illustrate DataParallelismTest on X86( 2 Cores), X64 machines (8 cores)

7) Parallel LINQ
http://www.codeproject.com/Articles/161434/Task-Parallel-Library-of-n

7) Task Parallelism

5) Thread Pools - Deep Dive
http://www.danielmoth.com/Blog/New-And-Improved-CLR-4-Thread-Pool-Engine.aspx

Software Downloads

PRISM 5 Download
http://www.microsoft.com/en-us/download/confirmation.aspx?id=42537

MDT

--WorkUnit.cs

    public class WorkUnit
    {
        public int Id { get; set; }
        public string Code { get; set; }
        public string Description { get; set; }
        public string FilePath { get; set; }
        public int ExecutionTimeForWorkUnit { get; set; }
    }

--WorkUnitManager.cs
    public class WorkUnitManager    {
        private List<WorkUnit> _tasks;
        public WorkUnitManager()
        {
        }

        public IList<WorkUnit> GenerateWorkUnits()
        {
            var filePath = ConfigurationManager.AppSettings["feedLocation"];
            var workUnits = new List<WorkUnit>();
            for (var i = 0; i < 2; i++)
            {
                var workUnit = new WorkUnit { Id = i, Code = "workUnit" + i, FilePath = filePath + @"\ModelData" + i + ".csv", ExecutionTimeForWorkUnit = ( i==0) ? 1000 :  2000 * i};
                workUnits.Add(workUnit);
            }
            return workUnits;
        }

        public bool Process(IList<WorkUnit> workUnits )
        {
            try
            {
                WaitHandle[] waitHandles = new WaitHandle[workUnits.Count];
                for (var i = 0; i < workUnits.Count; i++)
                    waitHandles[i] = new AutoResetEvent(false) ;
                int index = 0;
                foreach (var unit in workUnits)
                {
                    //starting the thread
                    int index1 = index;
                    WorkUnit unit1 = unit;
                    var thread = new Thread(parameter =>
                    {
                        ProcessWorkUnit(unit1);
                        ((AutoResetEvent) waitHandles[index1]).Set();
                    });
                    thread.Start(unit);
                    index++;
                }
                WaitHandle.WaitAll(waitHandles);
                return true;
            }
            catch(Exception exception)
            {
                return false;
            }
        }

        private void ProcessWorkUnit(WorkUnit workUnit)
        {
            //Thread.Sleep(workUnit.ExecutionTimeForWorkUnit);
            ModelManager modelManager = new ModelManager();
            modelManager.ProcessModel(workUnit);
            var threadId = Thread.CurrentThread.ManagedThreadId;
            string logMessage = string.Format("Work Unit processed  => {0} after {1} ms. on thread {2} ", workUnit.Code, workUnit.ExecutionTimeForWorkUnit, threadId);
            ApplicationLogger.Instance.Log(typeof(WorkUnitManager), "------------------------------Start : Work Unit Information---------------------------", LogLevel.Info);
            ApplicationLogger.Instance.Log(typeof(WorkUnitManager), logMessage, LogLevel.Info);
            ApplicationLogger.Instance.Log(typeof(WorkUnitManager), "------------------------------End : Work Unit Information---------------------------", LogLevel.Info);
        }
    }

--
public class Model
    {
        public int Id { get; set; }
        public string Name { get; set; }
        public string Description { get; set; }
        public string InvestmentCode { get; set; }
        public double FxRate { get; set; }
        public double DailyPnl { get; set; }
    }

public class ModelValidation
    {
        public string Id { get; set; }
        public int ModelId { get; set; }
        public bool IsValid { get; set; }
        public string Exception { get; set; }
    }

--ModelTransaction.cs
public class ModelTransaction
    {
        public int Id { get; set; }
        public string Name { get; set; }
        public string Description { get; set; }
        public string InvestmentCode { get; set; }
        public int FundId { get; set; }
        public int InvestmentId { get; set; }
        public double FxRate { get; set; }
        public double DailyPnl { get; set; }
        public double RepPnl { get; set; }
    }

--
public enum LogLevel
    {
        Debug,
        Info,
        Warn,
        Error
    }

    public sealed class ApplicationLogger
    {
        private static readonly ApplicationLogger _instance = new ApplicationLogger();
        private static ILog logger;

        static ApplicationLogger()
        {
            XmlConfigurator.Configure();
        }

        public static ApplicationLogger Instance
        {
            get { return _instance; }
        }

        public void Log(Type type, string message, LogLevel logLevel)
        {
            logger = LogManager.GetLogger(type);
            switch (logLevel)
            {
                case LogLevel.Debug:
                    logger.Debug(message);
                    break;
                case LogLevel.Warn:
                    logger.Warn(message);
                    break;
                case LogLevel.Info:
                    logger.Info(message);
                    break;
                case LogLevel.Error:
                    logger.Error(message);
                    break;
            }
        }
    }

--ModelManager.cs
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CodeSnippets.Shared;
using CodeSnippets.Shared.Domain;

namespace CodeSnippets.Domain
{
    public class ModelManager : IModelManager
    {
        private IList<Model> _models;
        private IList<ModelTransaction> _modelTransactions;
        public ModelManager()
        {
         
        }

        public bool ProcessModel(WorkUnit workUnit)
        {
            Stopwatch stopwatch = new Stopwatch();
            // Begin timing
            stopwatch.Start();

            var modelRepository = new ModelRepository();

            //Load Models
            _models = modelRepository.Load(workUnit.FilePath);
            stopwatch.Stop();
            string logMessage = string.Format(" Operation : Load List<Model> : {0}, Processing Time : {1}", workUnit.FilePath, stopwatch.Elapsed.Milliseconds);
            ApplicationLogger.Instance.Log(typeof(ModelRepository), logMessage, LogLevel.Info);
            stopwatch.Reset();

            //Transform Models
            stopwatch.Start();
            _modelTransactions = modelRepository.Transform(_models);
            stopwatch.Stop();
            logMessage = string.Format("    Operation : Transform List<Model> : {0}, Processing Time : {1}", workUnit.FilePath, stopwatch.Elapsed.Milliseconds);
            ApplicationLogger.Instance.Log(typeof(ModelRepository), logMessage, LogLevel.Info);
            stopwatch.Reset();

            //Compute Models
            stopwatch.Start();
            modelRepository.Compute(_modelTransactions);
            stopwatch.Stop();
            logMessage = string.Format("    Operation : Compute RepPnL : {0}, Processing Time : {1}", workUnit.FilePath, stopwatch.Elapsed.Milliseconds);
            ApplicationLogger.Instance.Log(typeof(ModelRepository), logMessage, LogLevel.Info);
            stopwatch.Reset();

            //Enrich Models
            stopwatch.Start();
            modelRepository.Enrich(_modelTransactions);
            stopwatch.Stop();
            logMessage = string.Format("    Operation : Enrich Model Transactions : {0}, Processing Time : {1}", workUnit.FilePath, stopwatch.Elapsed.Milliseconds);
            ApplicationLogger.Instance.Log(typeof(ModelRepository), logMessage, LogLevel.Info);
            stopwatch.Reset();
            return true;
        }

        public IList<Model> GetModels()
        {
            return _models;
        }

        public IList<ModelTransaction> GetModelTransactions()
        {
            return _modelTransactions;
        }
    }
}

--ModelRepository.cs
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CodeSnippets.Shared;
using CodeSnippets.Shared.Domain;

namespace CodeSnippets.Domain
{
    public class ModelRepository
    {
        private readonly List<ModelValidation> _modelValidations;
        private readonly IRepository<Fund> _fundRepository;
        private readonly IRepository<Investment> _investmentRepository;

        public ModelRepository()
        {
            _fundRepository = new FundRepository();
            _investmentRepository = new InvestmentRepository();
            _modelValidations = new List<ModelValidation>();
        }

        public IList<Model> Load(string filePath)
        {
            var models = new List<Model>();
            try
            {
                int rowIndex = 0;
                using (var readFile = new StreamReader(filePath))
                {
                    string line;
                    while ((line = readFile.ReadLine()) != null)
                    {
                        string[] row = line.Split(',');
                        if (rowIndex != 0)
                        {
                            var model = new Model
                            {
                                Name = row[0],
                                Description = row[1],
                                InvestmentCode = row[2],
                                FxRate = Convert.ToDouble(row[3]),
                                DailyPnl = Convert.ToDouble(row[4])
                            };
                            models.Add(model);
                        }
                        rowIndex++;
                    }
                }
            }
            catch (Exception e)
            {
                string logMessage = "ModelRepository > Load " + "[" + e.StackTrace + "]";
                ApplicationLogger.Instance.Log(typeof(ModelRepository),  logMessage, LogLevel.Error);
            }
            return models;
        }

        public IList<ModelTransaction> Transform(IList<Model> models)
        {
            return models.Select(model => new ModelTransaction
            {
                Name = model.Name,
                Description = model.Description,
                InvestmentCode = model.InvestmentCode,
                FxRate = model.FxRate,
                DailyPnl = model.DailyPnl
            }).ToList();
        }

        public IList<ModelTransaction> Compute(IList<ModelTransaction> modelTransactions)
        {
            foreach (var modelTransaction in modelTransactions)
                modelTransaction.RepPnl = modelTransaction.FxRate*modelTransaction.DailyPnl;
            return modelTransactions;
        }

        public void Enrich(IList<ModelTransaction> modelTransactions)
        {
            List<Fund> funds = _fundRepository.GetAll().ToList();
            List<Investment> investments = _investmentRepository.GetAll().ToList();
            var enrichQuery = (from i in investments
                               join f in funds on i.FundId equals f.Id
            select new { FundName = f.Name, FundID = f.Id, InvestmentCode = i.Code, InvestmentId = i.Id, InvestmentName = i.Name });
            foreach (var modelTransaction in modelTransactions)
            {
                var fundQuery = enrichQuery.FirstOrDefault(e => e.InvestmentCode == modelTransaction.InvestmentCode);
                modelTransaction.FundId = (fundQuery == null) ? -1 : fundQuery.FundID;
                var investmentQuery = enrichQuery.FirstOrDefault(e => e.InvestmentCode == modelTransaction.InvestmentCode);
                modelTransaction.InvestmentId = (investmentQuery == null) ? -1 : investmentQuery.InvestmentId;
            }
        }

        public void Validate(IList<ModelTransaction> modelTransactions)
        {
            throw new NotImplementedException();
        }

        public void Persist(IList<ModelTransaction> modelTransactions)
        {
            throw new NotImplementedException();
        }
    }
}

--

<?xml version="1.0" encoding="utf-8" ?>

<configuration>

  <configSections>

    <section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />

  </configSections>

  <appSettings>

    <add key="feedLocation" value="C:\Data\Professional\TechnologySkills\Software Engg Principles\SourceCode\MultiThreading\ThreadPool\QueueWorkUserItem\Feeds" />

  </appSettings>

  <log4net>

    <appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender" >

      <param name="File" value="C:\Data\Professional\TechnologySkills\Software Engg Principles\SourceCode\MultiThreading\ThreadPool\QueueWorkUserItem\Logs\MDT.UnitTests.log" />

      <param name="AppendToFile" value="true" />

      <rollingStyle value="Size" />

      <maxSizeRollBackups value="2" />

      <maximumFileSize value="10MB" />

      <staticLogFileName value="true" />

      <lockingModel type="log4net.Appender.FileAppender+MinimalLock" />

      <layout type="log4net.Layout.PatternLayout">

        <param name="ConversionPattern"

             value="%-5p%d{yyyy-MM-dd hh:mm:ss} – %m%n" />

        <conversionPattern value="%date %logger  >> %message%newline" />

      </layout>

    </appender>

    <root>

      <level value="ALL" />

      <appender-ref ref="LogFileAppender" />

    </root>

  </log4net>

</configuration>

--InvestmentRepository
 public IEnumerable<Investment> GetAll()
        {
            return new List<Investment>
            {
                new Investment(){ Name = "MS Boston", Code = "MSB", Id=1, FundId = 1},
                new Investment(){ Name = "MS Woodbarn", Code = "MSW", Id=2, FundId = 1 },
                new Investment(){ Name = "MS Cash Eq", Id=3, Code = "MCE", FundId = 2 },
                new Investment(){ Name = "MS Cash Eq1", Id=4, Code = "MCE1", FundId = 2 },
                new Investment(){ Name = "MS Cash Eq2", Id=5, Code = "MCE2", FundId = 2 },
                new Investment(){ Name = "MS Comm", Id=6, Code = "MSCOM", FundId = 3 }
            };
        }

--FundRepositoy
        public IEnumerable<Fund> GetAll()
        {
            return new List<Fund>
            {
                new Fund(){ Name = "MS REF 1", Code = "MSR1",  Id=1 },
                new Fund(){ Name = "MS REF 2", Code = "MSR2", Id=2 },
                new Fund(){ Name = "MS REF 3", Code = "MSR#", Id=3 }
            };
        }
---Feeds
Name,Description,InvestmentCode, FXRate,DailyPnl
MCE2 test,MCE2 test DESCR,MCE2,1.0,4575
MSCOM test,MSCOM test DESCR,MSCOM,2,357
MCE test,MCE test DESCR,MCE,.6,34
MCE1 test,MCE1 test DESCR,MCE1, 2.5, 1000
MSE test,MSE test DESCR,MSE,3,489
TSE test,TSE test DESCR,TSE,4,900
MSB Test1,MSB Test1 DESCR,MSB, 2.4,12
MSW test,MSW test DESCR,MSW, 3.4,200
MCE test,MCE test DESCR,MCE, 5.6, 34
MCE1 test,MCE1 test DESCR,MCE1, 3.2,1000

   WorkUnitManager workUnitManager = new WorkUnitManager();
            IList<WorkUnit> workUnits = workUnitManager.GenerateWorkUnits();
            foreach (var workUnit in workUnits)
            {
                Console.WriteLine(" Work Unit {0}  ", workUnit.Code);
            }
            workUnitManager.Process(workUnits);

Multi -threading Querirs

Difference between ManaulResetEvent and AutoResetEvent 

• Difference between a tollbooth and a door. The ManualResetEvent is the door, which needs to be closed (reset)
• AutoResetEvent executes WaitOne() and Reset() as a single atomic operation..
• An AutoResetEvent will only allow one single waiting thread to continue. A ManualResetEvent on the other hand will keep allowing threads, several at the same time even, to continue until you tell it to stop (Reset it).

Exception Handling across several threads

http://www.codeproject.com/Articles/152765/Task-Parallel-Library-of-n

http://www.yoda.arachsys.com/csharp/threads/threadpool.shtml

Code Quirks

Can Func's point to instance methods ?
Whats the difference between Func and Delegates ?
Difference between lock and Thread.QueueWorkUserItem
Can delegates point to instance methods ?

-----
using System;
using System.Collections.Generic;
using System.Linq;

namespace LinqExamples
{
    #region Data

    class Product
    {
        public string Name { get; set; }
        public int CategoryID { get; set; }
    }

    class Category
    {
        public string Name { get; set; }
        public int ID { get; set; }
    }

    public class Student
    {
        public string Name { get; set; }
        public string Standard { get; set; }
        public string Status { get; set; }

        public Student(string name, string standard, string status)
        {
            this.Name = name;
            this.Standard = standard;
            this.Status = status;
        }
    }
    #endregion

    class Program
    {
        static void Main(string[] args)
        {
            /*
            var users = new List<User>
            {
                new User{Id = 1, FirstName = "Sam"},
                new User{Id = 2, FirstName = "Dave"},
                new User{Id = 3, FirstName = "Julia"}
            };

            var groups = new List<Group>
            {
                new Group{Id = 1, Name = "NJHawks"},
                new Group{Id = 2, Name = "NYJets"}
            };

            var userGroups = new List<UserGroup>
            {
                new UserGroup{UserId = 1, GroupId=1},
                new UserGroup{UserId = 1, GroupId=2},
                new UserGroup{UserId = 2, GroupId=2},
                new UserGroup{UserId = 3, GroupId=1},
                new UserGroup{UserId = 3, GroupId=2}
            };


            var innerJoinQuery = from u in users
                                 join ug in userGroups on u.Id equals ug.UserId
                                 join g in groups on ug.GroupId equals g.Id
                                 where u.FirstName == "Sam"
                                 select new {UserId = u.Id, u.FirstName, GroupName = g.Name};

         
          foreach (var item in innerJoinQuery)
           {
               Console.WriteLine("FirstName : {0} ,UserId : {1}, GroupId : {2} ", item.UserId,  item.FirstName,  item.GroupName);
           }

            */

            //Updating Linq Objects
            /*var arrStudents = new Student[]
                                    {
                                        new Student("AAA","1","A"),
                                        new Student("BBB","2","B"),
                                        new Student("CCC","3","C"),
                                        new Student("DDD","4","D"),
                                        new Student("EEE","5","E"),
                                        new Student("FFF","6","F"),
                                        new Student("GGG","7","G"),
                                        new Student("HHH","8","H"),
                                        new Student("III","9","I")
                                    };

            Console.WriteLine("Before update");
            foreach (Student student in arrStudents)
            {
                Console.WriteLine(String.Format("{0},{1},{2}", student.Name, student.Standard, student.Status));
            }

            var query = (from stud in arrStudents
                         where stud.Name == "BBB"
                         select stud)
                        .Update(st => { st.Standard = "0"; st.Status = "X"; });

            Console.WriteLine("After update");
            foreach (Student _student in arrStudents)
            {
                Console.WriteLine(String.Format("{0},{1},{2}", _student.Name, _student.Standard, _student.Status));
            }*/

            /*Update Collections */
            /*Create data repository*/
            var fundFamilies = new List<FundFamily>
                                   {
                                       new FundFamily {Id = 1, Code = "RE"},
                                       new FundFamily {Id = 2, Code = "NON-RE"},
                                       new FundFamily {Id = 3, Code = "PEQ"}
                                   };

            var funds = new List<Fund>
                            {
                                new Fund {Id = 1, Code = "MSREF1", FundFamilyId = 1},
                                new Fund {Id = 2, Code = "MSREF2", FundFamilyId = 1},
                                new Fund {Id = 3, Code = "EQ1", FundFamilyId = 2}
                            };
            var modelDatas = new List<ModelData>
                                 {
                                     new ModelData()
                                         {ModelCode = "BSP", ModelName = "Boston Seaport", FundCode = "MSREF1"},
                                     new ModelData()
                                         {ModelCode = "WBS", ModelName = "Woodbrook Sea", FundCode = "MSREF1"},
                                     new ModelData()
                                         {ModelCode = "EQTL", ModelName = "Private Tryport", FundCode = "EQ1"},
                                     new ModelData()
                                         {ModelCode = "EQTL1", ModelName = "Private Tryport2", FundCode = "EQ1"}
                                 };

            var query = from f in funds
                        join ff in fundFamilies on f.FundFamilyId equals ff.Id
                        select new {FundFamilyId = ff.Id, FundFamilyCode = ff.Code, Fundid = f.Id, FundCode = f.Code};
            //foreach (var bp in query)
            //    Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", bp.FundFamilyId, bp.FundFamilyCode, bp.Fundid, bp.FundCode) ;

            //Create ModelTransaction
            var modelTransactions = new List<ModelTransaction>
                                        {
                                            new ModelTransaction()
                                                {ModelCode = "BSP", ModelName = "Boston Seaport", FundCode = "MSREF1"},
                                            new ModelTransaction()
                                                {ModelCode = "WBS", ModelName = "Woodbrook Sea", FundCode = "MSREF1"},
                                            new ModelTransaction()
                                                {ModelCode = "EQTL", ModelName = "Private Tryport", FundCode = "EQ1"},
                                            new ModelTransaction()
                                                {ModelCode = "EQTL1", ModelName = "Private Tryport2", FundCode = "EQ1"}
                                        };


            Console.WriteLine("--Before  Update----");
            foreach (ModelTransaction modelTransaction in modelTransactions)
                Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", modelTransaction.FundFamilyId,
                                  modelTransaction.FundCode, modelTransaction.FundId, modelTransaction.FundFamilyId);

            Console.WriteLine("--After Update----");
            var metaDataQuery = (from f in funds
                                 join ff in fundFamilies on f.FundFamilyId equals ff.Id
                                 select new {FundCode = f.Code, f.FundFamilyId, FundId = f.Id});
            //.Update(mt => { mt.FundFamilyId = 1 ; mt.FundId = 2; });
         
            //var updateModelTransactionsQuery = (
            //                                       from m in modelTransactions
            //                                       join f in funds on m.FundCode equals f.Code
            //                                       join ff in fundFamilies on f.FundFamilyId equals ff.Id
            //                                       select m);
            //foreach (var m in updateModelTransactionsQuery)
            //{

            //}
            foreach (ModelTransaction modelTransaction in modelTransactions)
            {
                int fundId= metaDataQuery.FirstOrDefault(i => i.FundCode == modelTransaction.FundCode).FundId;
                int fundFamilyId= metaDataQuery.FirstOrDefault(i => i.FundCode == modelTransaction.FundCode).FundFamilyId;
                modelTransaction.FundId = fundId;
                modelTransaction.FundFamilyId = fundFamilyId;
            }

             foreach (ModelTransaction modelTransaction in modelTransactions)
                Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", modelTransaction.FundFamilyId, modelTransaction.FundCode, modelTransaction.FundId, modelTransaction.FundFamilyId);
           
     
            Console.ReadLine();
        }
    }

 
    public static class UpdateExtensions
    {
        public delegate void Func<TArg0>(TArg0 element);

        public static int Update<TSource>(this IEnumerable<TSource> source, Func<TSource> update)
        {
            if (source == null) throw new ArgumentNullException("source");
            if (update == null) throw new ArgumentNullException("update");
            if (typeof(TSource).IsValueType)
                throw new NotSupportedException("value type elements are not supported by update.");

            int count = 0;
            foreach (TSource element in source)
            {
                update(element);
                count++;
            }
            return count;
        }
    }
}

Multi-Threaded DataManager - Design

TaskCategory
File
TSQLQuery
StoredProc
InMemory ( .NET Classes)

TaskOperationType : Will define the operation
FileExportToTask
FileParseTask
FileGenerationTask
FileToStoredProcedureTask
EmailTask
ExecuteStoredProcedureTask
SQLBulkCopyTask


TaskCategoryMetaDataType : Meta Data Attributes for a  Task Source Type
FileExportTask : sourcefile, destFile path, etc
DatabaseOperations : storedProd Name, parameters, etc

TaskCategoryMetaData : Meta Data Attributes Values that relate a Task source Type Id
FileExportTask : sourcefile, destFile path, etc
DatabaseOperations : storedProd Name, parameters, etc

TaskInstance
TaskOperationTypeId
FileSourceTaskCategoryMetaDetaiI    ( TaskCategoryMetaDataId )
FileDestinationTaskCategoryMetaDetaiI    ( TaskCategoryMetaDataId )

--Workflow
WorkflowSteps -- Predefined chain of Tasks
ValidationStep DataExtractionStep is a combination of FileExportTask and SqlBulkCopyTask

WorkUnit 
WorkflowStepId

WorkflowStepDependency (?)

WorkFlowInstance :
list of Tasks in a Queue

WorkflowInstanceHistory

--Dao
FileDao
Read a Flat File and get Reader

SqlBulkUpload

---what can be implemented ?
1) Read CSV Files in a multi-threaded fashion and execute simultaneously do sql Bulk copy into the database.
2) As a Start implement the following FileExportTask, SQLBulkCopyTask

How can it be implemented ?

1. WorkflowBroker will read and write from a syncQueue. 
2. Producer Consumer Queue http://www.albahari.com/threading/part2.aspx#_AutoResetEvent 

--------------

var fundFamilies = new List<FundFamily>

                                   {

                                       new FundFamily {Id = 1, Code = "RE"},

                                       new FundFamily {Id = 2, Code = "NON-RE"},

                                       new FundFamily {Id = 3, Code = "PEQ"}

                                   };

            var funds = new List<Fund>

                            {

                                new Fund {Id = 1, Code = "MSREF1", FundFamilyId = 1},

                                new Fund {Id = 2, Code = "MSREF2", FundFamilyId = 1},

                                new Fund {Id = 3, Code = "EQ1", FundFamilyId = 2}

                            };

            var modelDatas = new List<ModelData>

                                 {

                                     new ModelData()

                                         {ModelCode = "BSP", ModelName = "Boston Seaport", FundCode = "MSREF1"},

                                     new ModelData()

                                         {ModelCode = "WBS", ModelName = "Woodbrook Sea", FundCode = "MSREF1"},

                                     new ModelData()

                                         {ModelCode = "EQTL", ModelName = "Private Tryport", FundCode = "EQ1"},

                                     new ModelData()

                                         {ModelCode = "EQTL1", ModelName = "Private Tryport2", FundCode = "EQ1"}

                                 };

            var query = from f in funds

                        join ff in fundFamilies on f.FundFamilyId equals ff.Id

                        select new {FundFamilyId = ff.Id, FundFamilyCode = ff.Code, Fundid = f.Id, FundCode = f.Code};

            //foreach (var bp in query)

            //    Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", bp.FundFamilyId, bp.FundFamilyCode, bp.Fundid, bp.FundCode) ;

            //Create ModelTransaction 

            var modelTransactions = new List<ModelTransaction>

                                        {

                                            new ModelTransaction()

                                                {ModelCode = "BSP", ModelName = "Boston Seaport", FundCode = "MSREF1"},

                                            new ModelTransaction()

                                                {ModelCode = "WBS", ModelName = "Woodbrook Sea", FundCode = "MSREF1"},

                                            new ModelTransaction()

                                                {ModelCode = "EQTL", ModelName = "Private Tryport", FundCode = "EQ1"},

                                            new ModelTransaction()

                                                {ModelCode = "EQTL1", ModelName = "Private Tryport2", FundCode = "EQ1"}

                                        };

            Console.WriteLine("--Before  Update----");

            foreach (ModelTransaction modelTransaction in modelTransactions)

                Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", modelTransaction.FundFamilyId,

                                  modelTransaction.FundCode, modelTransaction.FundId, modelTransaction.FundFamilyId);

            Console.WriteLine("--After Update----");

            var metaDataQuery = (from f in funds

                                 join ff in fundFamilies on f.FundFamilyId equals ff.Id

                                 select new {FundCode = f.Code, f.FundFamilyId, FundId = f.Id});

            //.Update(mt => { mt.FundFamilyId = 1 ; mt.FundId = 2; });

           

            //var updateModelTransactionsQuery = (

            //                                       from m in modelTransactions

            //                                       join f in funds on m.FundCode equals f.Code

            //                                       join ff in fundFamilies on f.FundFamilyId equals ff.Id

            //                                       select m);

            //foreach (var m in updateModelTransactionsQuery)

            //{

            //}

            foreach (ModelTransaction modelTransaction in modelTransactions)

            {

                int fundId= metaDataQuery.FirstOrDefault(i => i.FundCode == modelTransaction.FundCode).FundId;

                int fundFamilyId= metaDataQuery.FirstOrDefault(i => i.FundCode == modelTransaction.FundCode).FundFamilyId;

                modelTransaction.FundId = fundId;

                modelTransaction.FundFamilyId = fundFamilyId;

            }

             foreach (ModelTransaction modelTransaction in modelTransactions)

                Console.WriteLine("Funds info  - {0}, {1}, {2}, {3}", modelTransaction.FundFamilyId, modelTransaction.FundCode, modelTransaction.FundId, modelTransaction.FundFamilyId);


==============
public class WorkUnit
    {
        public int Id { get; set; }
        public string Code { get; set; }
        public string Name { get; set; }
    }

 public class WorkUnitManager    {
        private List<WorkUnit> _tasks;
        public WorkUnitManager()
        {
        }

        public bool ProcessModel(Model model)
        {
            return true;
        }
    }

 public static void ProcessWorkUnit(Object stateInfo)
        {
            WorkUnit workUnit = (WorkUnit) stateInfo;
            Thread.Sleep(1000);
            Console.WriteLine(workUnit.Name);
        }

Main ()
  var workUnits = new List<WorkUnit>();
            for (int i = 0; i < 100; i++){
                var workUnit = new WorkUnit {Id = i, Name = "work Unit " + i};
                workUnits.Add(workUnit);
            }

            // Queue the task and data.
            foreach (var workUnit in workUnits)
                ThreadPool.QueueUserWorkItem(new WaitCallback(ProcessWorkUnit), workUnit);  
         
           Console.WriteLine("Main thread does some work, then sleeps.");

            // If you comment out the Sleep, the main thread exits before
            // the ThreadPool task has a chance to run.  ThreadPool uses
            // background threads, which do not keep the application
            // running.  (This is a simple example of a race condition.)
            Thread.Sleep(1000);

            Console.WriteLine("Main thread exits.");

====Parallel Processing
Source => Models

Processing Tasks
<= Synch Operations
Load() => Load List<Model> from a .csv file [Future this can be pushed into a Queue]
Transform() => Transform List<Model> to List<ModelTransaction> ( Synch Operations but can be executed in data parallelism)

<= Parallel Operations
Enrich() => Enrich List<ModelTransaction>
Compute() => Compute DailyPnL, MTDPnL, YTDPnL
Validate() =>
Validate List<ModelTransaction> for Enrichment
Validate List<ModelTransaction> for Computations
Persist() =>
Write modelTransactions  using SQLBulkCopy

Target => ModelTransactions

=> Implement the following using Thread.QueueWorkUserItem
=> Implement the following using Task Parallelism
=> Plot the Performance Difference I/O, CPU / Memory utilization ....

ModelRepository
Load()
Transform()

TaskRepository
ComputeTask
EnrichTask
ValidateTask
PersistTask

Web API Notes

Whats WEBAPI ?

• ASP.NET Web API is a framework that makes it easy to build HTTP services that reach a broad range of clients, including browsers and mobile devices. 
• With WebAPI content negotiation, we can return data based on the client requests,  if the client is requesting the data to be returned as JSON or XML, the WebAPI framework deals with the request type and returns the data appropriately based on the media type. By default WebAPI provides JSON and XML based responses.
• The client can make a GET, PUT, POST, and DELETE request and get the WebAPI response appropriately.

Content Negotiation in WebAPI

• Content negotiation is the process of selecting the best representation for a given response when there are multiple representations available. The underling Web API framework implements the content negotiation and that is the reason why and how the client can request data with a specific media type.
• By default the Web API returns data in JSON forMat, however while requesting for a resource we can specify the media type to return so that the WebAPI knows what you are requesting for and select the proper formatter to output the data. 

How to implement WebAPI

• Create WebApiConfig.cs wherein we defines routes in the route tabl.e

routes.MapHttpRoute(

    name: "API Default",

    routeTemplate: "api/{controller}/{id}",

    defaults: new { id = RouteParameter.Optional }

);

• Create a Class which derives from ApiController. 
• Define the GET/PUT/POST/DELETE methods defined in the WebAPI controller map to the HTTP methods. 

Routing 

public class ProductsController : ApiController

{

    [AcceptVerbs("GET", "HEAD")]

    public Product FindProduct(id) { }

    // WebDAV method

    [AcceptVerbs("MKCOL")]

    public void MakeCollection() { }

}

WebAPI Hosting Options

WebAPI can be hosted on IIS/Self Hosted.

Parameter Binding in WebAPI

There are various rules for binding the parameters:

• "Simple" type: If the parameter is the simple type then it is a string convertible parameter that includes the preemptive data types such as Int, Double, Bool and so on with the Date Time, Decimal, string and so on. By default these are read from the URI.
• "Complex" type: If the parameter is a complex type then the Web API catches the value from the message body. It uses the media type formatters for catching the value from the body.

FromURI Binding : 

In the From URI binding we use the [FromUri] attribute. The [FromUri] attribute is inherited from the [ModelBinder] attribute. Let's see an example:

GET/products?pageNo=1&price=20&size=xxl

public class Product

    {

        public int pageNo { get; set; }

        public decimal Price { get; set; }

        public int size { get; set; }

    }

FromBody Binding

public HttpResponseMessage Post([FromBody] string Address)

Self Host

Uri myUri = new Uri(@"http://localhost:9999");

// Let have our configurations readY

HttpSelfHostConfiguration config = new HttpSelfHostConfiguration(myUri); 

HttpSelfHostServer server = new HttpSelfHostServer(config);

// Start listening 

server.OpenAsync().Wait();

// Wait for it :)

Console.WriteLine("WebApi hosted on " + myUri.AbsoluteUri +" It can be tested now");

Console.ReadLine();

WebAPI / JQuery Integration

Assuming the Web API is hosted on IIS at http://localhost:8080/API_SVC/api/EmployeeAPI

--Get Operation

function GetAllEmployees() {

        jQuery.support.cors = true;

        $.ajax({

            url: 'http://localhost:8080/API_SVC/api/EmployeeAPI',

            type: 'GET',

            dataType: 'json',            

            success: function (data) {                

                WriteResponse(data);

            },

            error: function (x, y, z) {

                alert(x + '\n' + y + '\n' + z);

            }

        });        

    }

--Post Operation

function AddEmployee() {

        jQuery.support.cors = true;

        var employee = {

            ID: $('#txtaddEmpid').val(),

            EmpName: $('#txtaddEmpName').val(),

            EmpDepartment: $('#txtaddEmpDep').val(),

            EmpMobile: $('#txtaddEmpMob').val()

        };       

        

        $.ajax({

            url: 'http://localhost:8080/API_SVC/api/EmployeeAPI',

            type: 'POST',

            data:JSON.stringify(employee),            

            contentType: "application/json;charset=utf-8",

            success: function (data) {

                WriteResponse(data);

            },

            error: function (x, y, z) {

                alert(x + '\n' + y + '\n' + z);

            }

        });

    }

WebAPI Security

1) Enable Windows Authentication

• web.config

<system.web>

    <authentication mode="Windows" />

</system.web>

2) Enable CORS Support

• Browser security prevents a web page from making AJAX requests to another domain. This restriction is called thesame-origin policy. 
• Cross Origin Resource Sharing (CORS) is a W3C standard that allows a server to relax the same-origin policy. Using CORS, a server can explicitly allow some cross-origin requests while rejecting others.
• Enable CORS in WebAPI
• Import Package Microsoft,AspNet.webApi.Cors
• in the WebApiConfig . register enable CORS 

public static void Register(HttpConfiguration config)

        {

            // New code

            config.EnableCors();

            config.Routes.MapHttpRoute(

                name: "DefaultApi",

                routeTemplate: "api/{controller}/{id}",

                defaults: new { id = RouteParameter.Optional }

            );

        }

    }

• Next Enable the CORS Attribute in the Controller;s

[EnableCors(origins: "http://myclient.azurewebsites.net", headers: "*", methods: "*")]
    public class TestController : ApiController
    {
        // Controller methods not shown...

    }

ASP.NET MVC Coding Snippets

Displaying a datagrid in ASP.NET MVC 
http://www.4guysfromrolla.com/articles/121510-1.aspx

C# Interview Questions

What is dynamic ? 

1. The dynamic type enables the operations in which it occurs to bypass compile-time type checking. Instead, these operations are resolved at run time. Thedynamic type simplifies access to COM APIs such as the Office Automation APIs, and also to dynamic APIs such as IronPython libraries, and to the HTML Document Object Model (DOM).
2. The type is a static type, but an object of type dynamic bypasses static type checking. 

E.g.

/ Before the introduction of dynamic.
((Excel.Range)excelApp.Cells[1, 1]).Value2 = "Name";

// After the introduction of dynamic, the access to the Value property and 
// the conversion to Excel.Range are handled by the run-time COM binder.
excelApp.Cells[1, 1].Value = "Name";

ASP.NET MVC Notes

What is the difference between ViewData, ViewBag and TempData?

• In order to pass data from controller to view and in next subsequent request, ASP.NET MVC framework provides different options i.e., ViewData, ViewBag and TempData. Both ViewBag and ViewData are used to communicate between controller and corresponding view. But this communication is only for server call, it becomes null if redirect occurs. So, in short, it's a mechanism to maintain state between controller and corresponding view.
• ViewData is a dictionary object while ViewBag is a dynamic property (a new C# 4.0 feature). ViewData being a dictionary object is accessible using strings as keys and also requires typecasting for complex types. On the other hand, ViewBag doesn't have typecasting and null checks.
• TempData is also a dictionary object that stays for the time of an HTTP Request. So, Tempdata can be used to maintain data between redirects, i.e., from one controller to the other controller.

What are the different ways of passing information from the controller to view ?

1. As a strongly typed model object. @Model IEnumerable<type>
2. As a dynamic type (using @model dynamic)
3. Using the ViewBag

How to return/post JSON from Controller ?
// set up a click handler for the button to get the list of old things
$("#get-oldthings").click(function () {
    $.post('<%= Url.Action("GetOldThings") %>', { minAge: 10 }, ParseThingList);
});

[HttpPost]
public ActionResult GetOldThings(int minAge)
{
    IOldThingRepository repository = new OldThingRepository();
    return Json(repository.GetOldThings(minAge));
}

[HttpGet]

public JsonResult GetOldThings(int minAge)
{
    IOldThingRepository repository = new OldThingRepository();
    return Json(repository.GetOldThings(minAge), JsonRequestBehavior.AllowGet);
} 

Explain ASP.NET MVC application life cycle ? 



Step 1 Fill route: - MVC requests are mapped to route tables which in turn specify which controller and action to be invoked. So if the request is the first request the first thing is to fill the route table with routes collection. This filling of route table happens in the global.asax file.

Step 2 Fetch route: - Depending on the URL sent “UrlRoutingModule” searches the route table to create “RouteData” object which has the details of which controller and action to invoke.

Step 3 Request context created: - The “RouteData” object is used to create the “RequestContext” object.

Step 4 Controller instance created: - This request object is sent to “MvcHandler” instance to create the controller class instance. Once the controller class object is created it calls the “Execute” method of the controller class.

Whats the difference between Html.RenderPartial() and Html.Partial() ? 

@Html.RenderPartial("~/Views/Shared/_Product.cshtml", product);

RenderPartial renders the HTML O/P directly in the HttpOutputStream while Partial() can be retreived into a variable

Difference between Html.RenderPartial(), Html.Partial(), Html.RenderAction(), Html.Action()

• There is a big difference between RenderAction and RenderPartial. RenderPartial will render a View on the same controller (or a shared one), while RenderAction will actually perform an entire cycle of MVC, 
• it will instantiate the controller (any controller you mention, not just the current one), it will execute the action, and it will then return and render the result. 
• RenderPartial is more like an include 

How can we do validations in MVC ?

• MVC is by using data annotations. Data annotations are nothing but attributes which can be applied on model properties

public class Customer
{
    [Required(ErrorMessage="Customer code is required")]
    public string CustomerCode
    {
        set;
        get;
    } 
}  

//Displaying Messages
<% using (Html.BeginForm("PostCustomer", "Home", FormMethod.Post))
{ %>
<%=Html.TextBoxFor(m => m.CustomerCode)%>
<%=Html.ValidationMessageFor(m => m.CustomerCode)%>
<input type="submit" value="Submit customer data" />

<%}%> 

What are the other data annotation attributes for validation in MVC?

//If you want to check string length, you can use StringLength.
[StringLength(160)]
public string FirstName { get; set; }

//In case you want to use a regular expression, you can use the RegularExpression attribute.
[RegularExpression(@"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}")]public string Email { get; set; }

//If you want to check whether the numbers are in range, you can use the Range attribute.
[Range(10,25)]public int Age { get; set; }

//compare the value of one field with another field, we can use the Compare attribute.
public string Password { get; set; }[Compare("Password")]public string ConfirmPass { get; set; }

//To get a particular error message , you can use the Errors collection.
var ErrMessage = ModelState["Email"].Errors[0].ErrorMessage;

//If you have created the model object yourself you can explicitly call TryUpdateModel in your controller to //check if the object is valid or not.

TryUpdateModel(NewCustomer);

//In case you want add errors in the controller you can use the AddModelError function.
ModelState.AddModelError("FirstName", "This is my server-side error.");

Difference between Web Services / Web API

	SOAP	WEB API
Size	Heavy weight because of complicated WSDL structure.	Light weight, only the necessary information is transferred.
Protocol	Independent of protocols.	Only for HTTP protocol
Formats	To parse SOAP message, the client needs to understand WSDL format. Writing custom code for parsing WSDL is a heavy duty task. If your client is smart enough to create proxy objects like how we have in .NET (add reference) then SOAP is easier to consume and call.	Output of WebAPI are simple string messages, JSON, simple XML format, etc. So writing parsing logic for that is very easy.
Principles	SOAP follows WS-* specification.	WebAPI follows REST principles.

What is Bundling and Minification ? 
Bundling
ASP.NET is adding a feature that makes it easy to “bundle” or “combine” multiple CSS and JavaScript files into fewer HTTP requests. This causes the browser to request a lot fewer files and in turn reduces the time it takes to fetch them. 

Minification 
This is a process that removes whitespace, comments and other unneeded characters from both CSS and JavaScript. The result is smaller files, which will download and load in a browser faster. 

How to implement Bundling in ASP.NET MVC ? 

1) In BundleConfig.cs, add the JS files you want bundle into a single entity in to the bundles collection. In the below code we are combining all the javascript JS files which exist in the Scripts folder as a single unit in to the bundle collection.
bundles.Add(new ScriptBundle("~/Scripts/MyScripts").Include(
"~/Scripts/*.js")); 

public  class BundleConfig
{
    public static void RegisterBundles(BundleCollection bundles)
    {
        bundles.Add(new ScriptBundle("~/Scripts/MyScripts").Include(
           "~/Scripts/*.js"));
        BundleTable.EnableOptimizations = true;
    }

2) Once you have combined your scripts into one single unit we then to include all the JS files into the view using the below code. The below code needs to be put in the ASPX or Razor view.

 <%= Scripts.Render("~/Scripts/MyScripts")  %>

7/16 =>
Creating Route/Custom Routes?
When a user enters this request, you want to return the blog entry that corresponds to the date 12/25/2009. In order to handle this type of request, you need to create a custom route.

E.g. /Archive/12-25-2009

routes.MapRoute( "Blog", // Route name "Archive/{entryDate}", // URL with parameters new { controller = "Archive", action = "Entry" } // Parameter defaults );

public class ArchiveController : Controller { public string Entry(DateTime entryDate) { return "You requested the entry from " + entryDate.ToString(); } }
The MVC framework is smart enough to convert the entry date from the URL into a DateTime value automatically. If the entry date parameter from the URL cannot be converted to a DateTime

Creating Route Constraint
route constraints to restrict the browser requests that match a particular route. You can use a regular expression to specify a route constraint.

routes.MapRoute( "Product", "Product/{productId}", new {controller="Product", action="Details"}, new {productId = @"\d+" } );

Creating a Custom Route Constraint

• A custom route constraint enables you to prevent a route from being matched unless some custom condition is matched.
• You implement a custom route constraint by implementing the IRouteConstraint interface.bool Match( HttpContextBase httpContext, Route route, string parameterName, RouteValueDictionary values, RouteDirection routeDirection )
• Usage 
• routes.MapRoute(

"Admin", "Admin/{action}", new {controller="Admin"}, new {isLocal=new LocalhostConstraint()} );

How Requests are processed by the Thread Pool

• In the web server, the .NET Framework maintains a pool of threads that are used to service ASP.NET requests. When a request arrives, a thread from the pool is dispatched to process that request. If the request is processed synchronously, the thread that processes the request is busy while the request is being processed, and that thread cannot service another request. T
• The number of threads in the thread pool is limited (the default maximum for .NET 4.5 is 5,000). 
• In large applications with high concurrency of  long-running requests, all available threads might be busy. This condition is known as thread starvation. When this condition is reached, the web server queues requests. If the request queue becomes full, the web server rejects requests with an HTTP 503 status (Server Too Busy). 
• Each new thread added to the thread pool has overhead (such as 1 MB of stack memory).
• When you’re doing asynchronous work, you’re not always using a thread. For example, when you make an asynchronous web service request, ASP.NET will not be using any threads between the async method call and the await.  

Choice Synchronous vs Asynchronous Method 

Use synchronous methods for the following conditions:

• The operations are simple or short-running.
• The operations are primarily CPU operations instead of operations that involve extensive disk or network overhead. Using asynchronous action methods on CPU-bound operations provides no benefits and results in more overhead.

Use asynchronous methods for the following conditions:

• You're calling services that can be consumed through asynchronous methods, and you're using .NET 4.5 or higher. UI threads are not blocked.
• The operations are network-bound or I/O-bound instead of CPU-bound.
• Parallelism is more important.
• You want to provide a mechanism that lets users cancel a long-running request.
• When the benefit of switching threads out weighs the cost of the context switch. In general, you should make a method asynchronous if the synchronous method waits on the ASP.NET request thread while doing no work.  By making the call asynchronous,  the ASP.NET request thread is not stalled doing no work while it waits for the web service request to complete.

Using Aysnchronous Methods in ASP.NET MVC

• The ASP.NET MVC 4 Controller class in combination .NET 4.5  enables you to write asynchronous action methods that return an object of type  Task<ActionResult>. 
• The .NET Framework 4.5 builds on this asynchronous support with  the await and async keywords that make working with Task objects.
• The await keyword is syntactical shorthand for indicating that a piece of code should asynchronously wait on some other piece of code.
• The async keyword represents a hint that you can use to mark methods as task-based asynchronous methods

E.g.
//Service Provider
public class DbHelper
{
    public async Task<List<Customer>> GetCustomerDataAsync()
    {
        NorthwindEntities db = new NorthwindEntities();
        var query = from c in db.Customers
                    orderby c.CustomerID ascending
                    select c;
        List<Customer> data = await query.ToListAsync();
        return data;
    }
}

//asynch Controller Methods
public async Task<ActionResult> IndexAsync()
{
    DbHelper helper = new DbHelper();
    List<Customer> data = await helper.GetCustomerDataAsync();
    return View(data);
}

Test Driven Development
ASP.NET MVC application provide a very good support for TDD because we can write unit test directly against our controllers without needing any web server

[TestClass]
    public class GroupControllerTest
    {

        [TestMethod]
        public void Index()
        {
            // Arrange
            var controller = new GroupController();

            // Act
            var result = (ViewResult)controller.Index();
        
            // Assert
            Assert.IsInstanceOfType(result.ViewData.Model, typeof(IEnumerable));
        }
    }

How to integrate JQuery and ASP.NET MVC ?
How to invoke Controller Methods from JQuery ?
CRUD base operations on a datagrid ?
What is WebAPI  ?
Brief Overview of WebAPI