LdrpInitializeProcess() 調(diào)用 void LdrpDetectDetour().這個(gè)名字不言自明.它不返回值而是初始化全局變量BOOLEAN LdrpDetourExist.這個(gè)例程首先檢查一些加載器關(guān)鍵例程是否被鉤住 - 目前有 5 個(gè)例程:

LdrpInitializeProcess() calls void LdrpDetectDetour(). This name speaks for itself. it does not return a value but initializes the global variable BOOLEAN LdrpDetourExist. This routine first checks whether some loader critical routines are hooked - currently these are 5 routines:

• NtOpenFile
• NtCreateSection
• NtQueryAttributesFile
• NtOpenSection
• NtMapViewOfSection

如果是 - LdrpDetourExist = TRUE;

如果沒(méi)有鉤住 - ThreadDynamicCodePolicyInfo 被查詢 - 完整代碼:

If not hooked - ThreadDynamicCodePolicyInfo is queried - full code:

void LdrpDetectDetour()
{
    if (LdrpDetourExist) return ;

    static PVOID LdrpCriticalLoaderFunctions[] = {
        NtOpenFile,
        NtCreateSection,
        ZwQueryAttributesFile,
        ZwOpenSection,
        ZwMapViewOfSection,
    };

    static M128A LdrpThunkSignature[5] = {
        //***
    };

    ULONG n = RTL_NUMBER_OF(LdrpCriticalLoaderFunctions);
    M128A* ppv = (M128A*)LdrpCriticalLoaderFunctions;
    M128A* pps = LdrpThunkSignature; 
    do
    {
        if (ppv->Low != pps->Low || ppv->High != pps->High)
        {
            if (LdrpDebugFlags & 5)
            {
                DbgPrint("!!! Detour detected, disable parallel loading
");
                LdrpDetourExist = TRUE;
                return;
            }
        }

    } while (pps++, ppv++, --n);

    BOOL DynamicCodePolicy;

    if (0 <= ZwQueryInformationThread(NtCurrentThread(), ThreadDynamicCodePolicyInfo, &DynamicCodePolicy, sizeof(DynamicCodePolicy), 0))
    {
        if (LdrpDetourExist = (DynamicCodePolicy == 1))
        {
            if (LdrpMapAndSnapWork)
            {
                WaitForThreadpoolWorkCallbacks(LdrpMapAndSnapWork, TRUE);//TpWaitForWork
                TpReleaseWork(LdrpMapAndSnapWork);//CloseThreadpoolWork
                LdrpMapAndSnapWork = 0;
                TpReleasePool(LdrpThreadPool);//CloseThreadpool
                LdrpThreadPool = 0;
            }
        }
    }
}

LdrpInitializeProcess() 調(diào)用 NTSTATUS LdrpEnableParallelLoading (ULONG LoaderThreads) - 作為 LdrpEnableParallelLoading(ProcessParameters->LoaderThreads):>

LdrpInitializeProcess() calls NTSTATUS LdrpEnableParallelLoading (ULONG LoaderThreads) - as LdrpEnableParallelLoading(ProcessParameters->LoaderThreads):

NTSTATUS LdrpEnableParallelLoading (ULONG LoaderThreads)
{
    LdrpDetectDetour();

    if (LoaderThreads)
    {
        LoaderThreads = min(LoaderThreads, 16);// not more than 16 threads allowed
        if (LoaderThreads <= 1) return STATUS_SUCCESS;
    }
    else
    {
        if (RtlGetSuiteMask() & 0x10000) return STATUS_SUCCESS; 
        LoaderThreads = 4;// default for 4 threads
    }

    if (LdrpDetourExist) return STATUS_SUCCESS;

    NTSTATUS status = TpAllocPool(&LdrpThreadPool, 1);//CreateThreadpool

    if (0 <= status)
    {
        TpSetPoolWorkerThreadIdleTimeout(LdrpThreadPool, -300000000);// 30 second idle timeout
        TpSetPoolMaxThreads(LdrpThreadPool, LoaderThreads - 1);//SetThreadpoolThreadMaximum 
        TP_CALLBACK_ENVIRON CallbackEnviron = { };
        CallbackEnviron->CallbackPriority = TP_CALLBACK_PRIORITY_NORMAL;
        CallbackEnviron->Size = sizeof(TP_CALLBACK_ENVIRON);
        CallbackEnviron->Pool = LdrpThreadPool;
        CallbackEnviron->Version = 3;

        status = TpAllocWork(&LdrpMapAndSnapWork, LdrpWorkCallback, 0, &CallbackEnviron);//CreateThreadpoolWork
    }

    return status;
}

創(chuàng)建了一個(gè)特殊的加載器線程池 - LdrpThreadPool，具有 LoaderThreads - 1 個(gè)最大線程.空閑超時(shí)設(shè)置為30秒(之后線程退出)并分配PTP_WORK LdrpMapAndSnapWork，然后在void LdrpQueueWork(LDRP_LOAD_CONTEXT* LoadContext)中使用.

A special loader thread pool is created - LdrpThreadPool, with LoaderThreads - 1 max threads. Idle timeout is set to 30 seconds (after which the thread exits) and allocated PTP_WORK LdrpMapAndSnapWork, which is then used in void LdrpQueueWork(LDRP_LOAD_CONTEXT* LoadContext).

并行加載器使用的全局變量:

Global variables used by the parallel loader:

HANDLE LdrpWorkCompleteEvent, LdrpLoadCompleteEvent;
CRITICAL_SECTION LdrpWorkQueueLock;
LIST_ENTRY LdrpWorkQueue = { &LdrpWorkQueue, &LdrpWorkQueue };


ULONG LdrpWorkInProgress;
BOOLEAN LdrpDetourExist;
PTP_POOL LdrpThreadPool;

PTP_WORK LdrpMapAndSnapWork;

enum DRAIN_TASK {
    WaitLoadComplete, WaitWorkComplete
};

struct LDRP_LOAD_CONTEXT
{
    UNICODE_STRING BaseDllName;
    PVOID somestruct;
    ULONG Flags;//some unknown flags
    NTSTATUS* pstatus; //final status of load
    _LDR_DATA_TABLE_ENTRY* ParentEntry; // of 'parent' loading dll
    _LDR_DATA_TABLE_ENTRY* Entry; // this == Entry->LoadContext
    LIST_ENTRY WorkQueueListEntry;
    _LDR_DATA_TABLE_ENTRY* ReplacedEntry;
    _LDR_DATA_TABLE_ENTRY** pvImports;// in same ordef as in IMAGE_IMPORT_DESCRIPTOR piid
    ULONG ImportDllCount;// count of pvImports
    LONG TaskCount;
    PVOID pvIAT;
    ULONG SizeOfIAT;
    ULONG CurrentDll; // 0 <= CurrentDll < ImportDllCount
    PIMAGE_IMPORT_DESCRIPTOR piid;
    ULONG OriginalIATProtect;
    PVOID GuardCFCheckFunctionPointer;
    PVOID* pGuardCFCheckFunctionPointer;
};

不幸的是 LDRP_LOAD_CONTEXT 未包含在已發(fā)布的 .pdb 文件中，因此我的定義僅包含部分名稱.

Unfortunately LDRP_LOAD_CONTEXT is not contained in published .pdb files, so my definitions include only partial names.

struct {
    ULONG MaxWorkInProgress;//4 - values from explorer.exe at some moment
    ULONG InLoaderWorker;//7a (this mean LdrpSnapModule called from worker thread)
    ULONG InLoadOwner;//87 (LdrpSnapModule called direct, in same thread as `LdrpMapAndSnapDependency`)
} LdrpStatistics;

// for statistics
void LdrpUpdateStatistics()
{
  LdrpStatistics.MaxWorkInProgress = max(LdrpStatistics.MaxWorkInProgress, LdrpWorkInProgress);
  NtCurrentTeb()->LoaderWorker ? LdrpStatistics.InLoaderWorker++ : LdrpStatistics.InLoadOwner++
}

在 TEB.CrossTebFlags - 現(xiàn)在存在 2 個(gè)新標(biāo)志:

In TEB.CrossTebFlags - now exist 2 new flags:

USHORT LoadOwner : 01; // 0x1000;
USHORT LoaderWorker : 01; // 0x2000;

最后 2 位是空閑的 (USHORT SpareSameTebBits : 02;//0xc000)

Last 2 bits is spare (USHORT SpareSameTebBits : 02; // 0xc000)

LdrpMapAndSnapDependency(LDRP_LOAD_CONTEXT* LoadContext) 包括以下代碼:

LDR_DATA_TABLE_ENTRY* Entry = LoadContext->CurEntry;
if (LoadContext->pvIAT)
{
    Entry->DdagNode->State = LdrModulesSnapping;
    if (LoadContext->PrevEntry)// if recursive call
    {
        LdrpQueueWork(LoadContext); // !!!
    }
    else
    {
        status = LdrpSnapModule(LoadContext);
    }
}
else
{
    Entry->DdagNode->State = LdrModulesSnapped;
}

所以，如果 LoadContext->PrevEntry(假設(shè)我們加載 user32.dll.在第一次調(diào)用 LdrpMapAndSnapDependency() 時(shí)，LoadContext->PrevEntry 將始終為 0(當(dāng) CurEntry 指向 user32.dll 時(shí))，但是當(dāng)我們遞歸調(diào)用 LdrpMapAndSnapDependency() 依賴 gdi32.dll - PrevEntry 將用于 user32.dll 和 CurEntry 用于 gdi32.dll)，我們不直接調(diào)用LdrpSnapModule(LoadContext);而是LdrpQueueWork(LoadContext);.

So, if LoadContext->PrevEntry (say we load user32.dll. In the first call to LdrpMapAndSnapDependency(), LoadContext->PrevEntry will be always 0 (when CurEntry points to user32.dll), but when we recursively call LdrpMapAndSnapDependency() for it dependency gdi32.dll - PrevEntry will be for user32.dll and CurEntry for gdi32.dll), we do not direct call LdrpSnapModule(LoadContext); but LdrpQueueWork(LoadContext);.

LdrpQueueWork() 很簡(jiǎn)單:

void LdrpQueueWork(LDRP_LOAD_CONTEXT* LoadContext)
{
    if (0 <= ctx->pstatus)
    {
        EnterCriticalSection(&LdrpWorkQueueLock);

        InsertHeadList(&LdrpWorkQueue, &LoadContext->WorkQueueListEntry);

        LeaveCriticalSection(&LdrpWorkQueueLock);

        if (LdrpMapAndSnapWork && !RtlGetCurrentPeb()->Ldr->ShutdownInProgress)
        {
            SubmitThreadpoolWork(LdrpMapAndSnapWork);//TpPostWork
        }
    }
}

我們將 LoadContext 插入到 LdrpWorkQueue 中，如果Parallel loader"已啟動(dòng) (LdrpMapAndSnapWork != 0) 而不是 ShutdownInProgress - 我們將工作提交到加載器池.但是即使池沒(méi)有初始化(比如因?yàn)?Detours 存在) - 也不會(huì)出現(xiàn)錯(cuò)誤 - 我們?cè)?LdrpDrainWorkQueue() 中處理這個(gè)任務(wù).

We insert LoadContext to LdrpWorkQueue and if "Parallel loader" is started (LdrpMapAndSnapWork != 0) and not ShutdownInProgress - we submit work to loader pool. But even if the pool is not initialized (say because Detours exist) - there will be no error - we process this task in LdrpDrainWorkQueue().

在工作線程回調(diào)中執(zhí)行:

In a worker thread callback, this is executed:

void LdrpWorkCallback()
{
    if (LdrpDetourExist) return;

    EnterCriticalSection(&LdrpWorkQueueLock);

    PLIST_ENTRY Entry = RemoveEntryList(&LdrpWorkQueue);

    if (Entry != &LdrpWorkQueue)
    {
        ++LdrpWorkInProgress;
        LdrpUpdateStatistics()
    }

    LeaveCriticalSection(&LdrpWorkQueueLock);

    if (Entry != &LdrpWorkQueue)
    {
        LdrpProcessWork(CONTAINING_RECORD(Entry, LDRP_LOAD_CONTEXT, WorkQueueListEntry), FALSE);
    }
}

我們只需從 LdrpWorkQueue 中彈出一個(gè)條目，將其轉(zhuǎn)換為 LDRP_LOAD_CONTEXT* (CONTAINING_RECORD(Entry, LDRP_LOAD_CONTEXT, WorkQueueListEntry)) 并調(diào)用 <代碼>void LdrpProcessWork(LDRP_LOAD_CONTEXT* LoadContext, BOOLEAN LoadOwner).

We simply popup an entry from LdrpWorkQueue, convert it to LDRP_LOAD_CONTEXT* (CONTAINING_RECORD(Entry, LDRP_LOAD_CONTEXT, WorkQueueListEntry)) and call void LdrpProcessWork(LDRP_LOAD_CONTEXT* LoadContext, BOOLEAN LoadOwner).

void LdrpProcessWork(LDRP_LOAD_CONTEXT* ctx, BOOLEAN LoadOwner)通常調(diào)用 LdrpSnapModule(LoadContext) 并在最后執(zhí)行下一個(gè)代碼:

void LdrpProcessWork(LDRP_LOAD_CONTEXT* ctx, BOOLEAN LoadOwner) in general calls LdrpSnapModule(LoadContext) and in the end the next code is executed:

if (!LoadOwner)
{
    EnterCriticalSection(&LdrpWorkQueueLock);
    BOOLEAN bSetEvent = --LdrpWorkInProgress == 1 && IsListEmpty(&LdrpWorkQueue);
    LeaveCriticalSection(&LdrpWorkQueueLock);
    if (bSetEvent) ZwSetEvent(LdrpWorkCompleteEvent, 0);
}

所以，如果我們不是LoadOwner(在工作線程中)，我們遞減LdrpWorkInProgress，如果LdrpWorkQueue 為空，則信號(hào)LdrpWorkCompleteEvent(LoadOwner 可以等待).

So, if we are not LoadOwner (in worked thread), we decrement LdrpWorkInProgress, and if LdrpWorkQueue is empty then signal LdrpWorkCompleteEvent (LoadOwner can wait on it).

最后，LdrpDrainWorkQueue()從LoadOwner(主線程)被調(diào)用到drain".工作隊(duì)列.它可以彈出并直接執(zhí)行由 LdrpQueueWork() 推送到 LdrpWorkQueue 的任務(wù)，但不會(huì)被工作線程彈出或因?yàn)椴⑿屑虞d器被禁用(在這種情況下LdrpQueueWork() 也推送 LDRP_LOAD_CONTEXT 但并沒(méi)有真正將工作發(fā)布到工作線程)，最后等待(如果需要)LdrpWorkCompleteEvent 或 LdrpLoadCompleteEvent 事件.

and finally, LdrpDrainWorkQueue() is called from LoadOwner (primary thread) to "drain" the WorkQueue. It can possible pop and directly execute tasks pushed to LdrpWorkQueue by LdrpQueueWork(), and yet is not popped by worked threads or because parallel loader is disabled (in this case LdrpQueueWork() also push LDRP_LOAD_CONTEXT but not really post work to worked thread), and finally wait (if need) on LdrpWorkCompleteEvent or LdrpLoadCompleteEvent events.

enum DRAIN_TASK {
    WaitLoadComplete, WaitWorkComplete
};

void LdrpDrainWorkQueue(DRAIN_TASK task)
{
    BOOLEAN LoadOwner = FALSE;

    HANDLE hEvent = task ? LdrpWorkCompleteEvent : LdrpLoadCompleteEvent;

    for(;;)
    {
        PLIST_ENTRY Entry;

        EnterCriticalSection(&LdrpWorkQueueLock);

        if (LdrpDetourExist && task == WaitLoadComplete)
        {
            if (!LdrpWorkInProgress)
            {
                LdrpWorkInProgress = 1;
                LoadOwner = TRUE;
            }
            Entry = &LdrpWorkQueue;
        }
        else
        {
            Entry = RemoveHeadList(&LdrpWorkQueue);

            if (Entry == &LdrpWorkQueue)
            {
                if (!LdrpWorkInProgress)
                {
                    LdrpWorkInProgress = 1;
                    LoadOwner = TRUE;
                }
            }
            else
            {
                if (!LdrpDetourExist)
                {
                    ++LdrpWorkInProgress;
                }
                LdrpUpdateStatistics();
            }
        }
        LeaveCriticalSection(&LdrpWorkQueueLock);

        if (LoadOwner)
        {
            NtCurrentTeb()->LoadOwner = 1;
            return;
        }

        if (Entry != &LdrpWorkQueue)
        {
            LdrpProcessWork(CONTAINING_RECORD(Entry, LDRP_LOAD_CONTEXT, WorkQueueListEntry), FALSE);
        }
        else
        {
            ZwWaitForSingleObject(hEvent, 0, 0);
        }
    }
}

void LdrpDropLastInProgressCount()
{
  NtCurrentTeb()->LoadOwner = 0;
  EnterCriticalSection(&LdrpWorkQueueLock);
  LdrpWorkInProgress = 0;
  LeaveCriticalSection(&LdrpWorkQueueLock);
  ZwSetEvent(LdrpLoadCompleteEvent);
}