Author: tthompson Date: Fri May 12 22:16:20 2017 New Revision: 74523
URL: http://svn.reactos.org/svn/reactos?rev=74523&view=rev Log: [NTFS] - Commit early results of a small restructuring effort: -Add a new member to the NTFS_ATTR_CONTEXT struct, a LARGE_MCB. This allows an attribute context to describe the cluster mapping of a non-resident file while allowing that mapping to change dynamically, without the context itself needing to be resized. This fixes problems which sometimes arose from resizing files. -Remove hacky code from NtfsWriteFile() for dealing with "stale" contexts. This fixes that issue. -Update SetDataAttributeLength(), PrepareAttributeContext(), ReleaseAttributeContext(), FreeClusters(), and AddRun() for the new member. -Update ReadAttribute() and WriteAttribute() to work with the changed structure. A very-soon-to-come commit will overhaul these functions so they'll operate directly on the LARGE_MCB, instead of converting to and from a packed list of data runs. (Sparse files are broken until then.) -Rename "RunBufferOffset" to "RunBufferSize" in several places where appropriate. -Fix, improve, and add some comments.
Modified: branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/attrib.c branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/mft.c branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/ntfs.h branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/rw.c
Modified: branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/attrib.c URL: http://svn.reactos.org/svn/reactos/branches/GSoC_2016/NTFS/drivers/filesyste... ============================================================================== --- branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/attrib.c [iso-8859-1] (original) +++ branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/attrib.c [iso-8859-1] Fri May 12 22:16:20 2017 @@ -261,39 +261,23 @@ ULONG RunLength) { NTSTATUS Status; - PUCHAR DataRun = (PUCHAR)&AttrContext->Record + AttrContext->Record.NonResident.MappingPairsOffset; int DataRunMaxLength; PNTFS_ATTR_RECORD DestinationAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + AttrOffset); - LARGE_MCB DataRunsMCB; ULONG NextAttributeOffset = AttrOffset + AttrContext->Record.Length; - ULONGLONG NextVBN = AttrContext->Record.NonResident.LowestVCN; - - // Allocate some memory for the RunBuffer + ULONGLONG NextVBN = 0; + PUCHAR RunBuffer; - ULONG RunBufferOffset = 0; + ULONG RunBufferSize;
if (!AttrContext->Record.IsNonResident) return STATUS_INVALID_PARAMETER;
- RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); - if (!RunBuffer) - { - DPRINT1("ERROR: Couldn't allocate memory for data runs!\n"); - return STATUS_INSUFFICIENT_RESOURCES; - } - - // Convert the data runs to a map control block - Status = ConvertDataRunsToLargeMCB(DataRun, &DataRunsMCB, &NextVBN); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to convert data runs to MCB (probably ran out of memory)!\n"); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); - return Status; - } + if (AttrContext->Record.NonResident.AllocatedSize != 0) + NextVBN = AttrContext->Record.NonResident.HighestVCN + 1;
// Add newly-assigned clusters to mcb _SEH2_TRY{ - if (!FsRtlAddLargeMcbEntry(&DataRunsMCB, + if (!FsRtlAddLargeMcbEntry(&AttrContext->DataRunsMCB, NextVBN, NextAssignedCluster, RunLength)) @@ -301,70 +285,65 @@ ExRaiseStatus(STATUS_UNSUCCESSFUL); } } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { - FsRtlUninitializeLargeMcb(&DataRunsMCB); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); _SEH2_YIELD(_SEH2_GetExceptionCode()); } _SEH2_END;
+ RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); + if (!RunBuffer) + { + DPRINT1("ERROR: Couldn't allocate memory for data runs!\n"); + return STATUS_INSUFFICIENT_RESOURCES; + }
// Convert the map control block back to encoded data runs - ConvertLargeMCBToDataRuns(&DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferOffset); + ConvertLargeMCBToDataRuns(&AttrContext->DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferSize);
// Get the amount of free space between the start of the of the first data run and the attribute end DataRunMaxLength = AttrContext->Record.Length - AttrContext->Record.NonResident.MappingPairsOffset;
// Do we need to extend the attribute (or convert to attribute list)? - if (DataRunMaxLength < RunBufferOffset) + if (DataRunMaxLength < RunBufferSize) { PNTFS_ATTR_RECORD NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset); DataRunMaxLength += Vcb->NtfsInfo.BytesPerFileRecord - NextAttributeOffset - (sizeof(ULONG) * 2);
// Can we move the end of the attribute? - if (NextAttribute->Type != AttributeEnd || DataRunMaxLength < RunBufferOffset - 1) + if (NextAttribute->Type != AttributeEnd || DataRunMaxLength < RunBufferSize - 1) { DPRINT1("FIXME: Need to create attribute list! Max Data Run Length available: %d\n", DataRunMaxLength); if (NextAttribute->Type != AttributeEnd) DPRINT1("There's another attribute after this one with type %0xlx\n", NextAttribute->Type); ExFreePoolWithTag(RunBuffer, TAG_NTFS); - FsRtlUninitializeLargeMcb(&DataRunsMCB); return STATUS_NOT_IMPLEMENTED; }
// calculate position of end markers - NextAttributeOffset = AttrOffset + AttrContext->Record.NonResident.MappingPairsOffset + RunBufferOffset; + NextAttributeOffset = AttrOffset + AttrContext->Record.NonResident.MappingPairsOffset + RunBufferSize; NextAttributeOffset = ALIGN_UP_BY(NextAttributeOffset, 8); - - // Write the end markers - NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset); - NextAttribute->Type = AttributeEnd; - NextAttribute->Length = FILE_RECORD_END;
// Update the length DestinationAttribute->Length = NextAttributeOffset - AttrOffset; AttrContext->Record.Length = DestinationAttribute->Length;
- // We need to increase the FileRecord size - FileRecord->BytesInUse = NextAttributeOffset + (sizeof(ULONG) * 2); - } - - // NOTE: from this point on the original attribute record will contain invalid data in it's runbuffer - // TODO: Elegant fix? Could we free the old Record and allocate a new one without issue? + // End the file record + NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset); + SetFileRecordEnd(FileRecord, NextAttribute, FILE_RECORD_END); + }
// Update HighestVCN DestinationAttribute->NonResident.HighestVCN = - AttrContext->Record.NonResident.HighestVCN = max(NextVBN - 1 + RunLength, + AttrContext->Record.NonResident.HighestVCN = max(NextVBN - 1 + RunLength, AttrContext->Record.NonResident.HighestVCN);
// Write data runs to destination attribute RtlCopyMemory((PVOID)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), RunBuffer, - RunBufferOffset); + RunBufferSize);
// Update the file record Status = UpdateFileRecord(Vcb, AttrContext->FileMFTIndex, FileRecord);
ExFreePoolWithTag(RunBuffer, TAG_NTFS); - FsRtlUninitializeLargeMcb(&DataRunsMCB);
NtfsDumpDataRuns((PUCHAR)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), 0);
@@ -567,7 +546,7 @@ DataRunLengthSize = GetPackedByteCount(Count, TRUE); DPRINT("%d bytes needed.\n", DataRunLengthSize);
- // ensure the next data run + end marker would be > Max buffer size + // ensure the next data run + end marker would be <= Max buffer size if (RunBufferOffset + 2 + DataRunLengthSize + DataRunOffsetSize > MaxBufferSize) { Status = STATUS_BUFFER_TOO_SMALL; @@ -698,17 +677,12 @@ NTSTATUS Status = STATUS_SUCCESS; ULONG ClustersLeftToFree = ClustersToFree;
- // convert data runs to mcb - PUCHAR DataRun = (PUCHAR)&AttrContext->Record + AttrContext->Record.NonResident.MappingPairsOffset; PNTFS_ATTR_RECORD DestinationAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + AttrOffset); - LARGE_MCB DataRunsMCB; ULONG NextAttributeOffset = AttrOffset + AttrContext->Record.Length; PNTFS_ATTR_RECORD NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset); - ULONGLONG NextVBN = AttrContext->Record.NonResident.LowestVCN; - - // Allocate some memory for the RunBuffer + PUCHAR RunBuffer; - ULONG RunBufferOffset = 0; + ULONG RunBufferSize = 0;
PFILE_RECORD_HEADER BitmapRecord; PNTFS_ATTR_CONTEXT DataContext; @@ -722,30 +696,13 @@ return STATUS_INVALID_PARAMETER; }
- RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); - if (!RunBuffer) - { - DPRINT1("ERROR: Couldn't allocate memory for data runs!\n"); - return STATUS_INSUFFICIENT_RESOURCES; - } - - // Convert the data runs to a map control block - Status = ConvertDataRunsToLargeMCB(DataRun, &DataRunsMCB, &NextVBN); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to convert data runs to MCB (probably ran out of memory)!\n"); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); - return Status; - } - + // Read the $Bitmap file BitmapRecord = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); if (BitmapRecord == NULL) { DPRINT1("Error: Unable to allocate memory for bitmap file record!\n"); - FsRtlUninitializeLargeMcb(&DataRunsMCB); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); return STATUS_NO_MEMORY; }
@@ -753,9 +710,7 @@ if (!NT_SUCCESS(Status)) { DPRINT1("Error: Unable to read file record for bitmap!\n"); - FsRtlUninitializeLargeMcb(&DataRunsMCB); ExFreePoolWithTag(BitmapRecord, TAG_NTFS); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); return 0; }
@@ -763,9 +718,7 @@ if (!NT_SUCCESS(Status)) { DPRINT1("Error: Unable to find data attribute for bitmap file!\n"); - FsRtlUninitializeLargeMcb(&DataRunsMCB); ExFreePoolWithTag(BitmapRecord, TAG_NTFS); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); return 0; }
@@ -777,9 +730,7 @@ { DPRINT1("Error: Unable to allocate memory for bitmap file data!\n"); ReleaseAttributeContext(DataContext); - FsRtlUninitializeLargeMcb(&DataRunsMCB); ExFreePoolWithTag(BitmapRecord, TAG_NTFS); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); return 0; }
@@ -792,7 +743,7 @@ { LONGLONG LargeVbn, LargeLbn;
- if (!FsRtlLookupLastLargeMcbEntry(&DataRunsMCB, &LargeVbn, &LargeLbn)) + if (!FsRtlLookupLastLargeMcbEntry(&AttrContext->DataRunsMCB, &LargeVbn, &LargeLbn)) { Status = STATUS_INVALID_PARAMETER; DPRINT1("DRIVER ERROR: FreeClusters called to free %lu clusters, which is %lu more clusters than are assigned to attribute!", @@ -806,7 +757,9 @@ // deallocate this cluster RtlClearBits(&Bitmap, LargeLbn, 1); } - FsRtlTruncateLargeMcb(&DataRunsMCB, AttrContext->Record.NonResident.HighestVCN); + FsRtlTruncateLargeMcb(&AttrContext->DataRunsMCB, AttrContext->Record.NonResident.HighestVCN); + + // decrement HighestVCN, but don't let it go below 0 AttrContext->Record.NonResident.HighestVCN = min(AttrContext->Record.NonResident.HighestVCN, AttrContext->Record.NonResident.HighestVCN - 1); ClustersLeftToFree--; } @@ -816,19 +769,27 @@ if (!NT_SUCCESS(Status)) { ReleaseAttributeContext(DataContext); - FsRtlUninitializeLargeMcb(&DataRunsMCB); ExFreePoolWithTag(BitmapData, TAG_NTFS); ExFreePoolWithTag(BitmapRecord, TAG_NTFS); - ExFreePoolWithTag(RunBuffer, TAG_NTFS); return Status; }
ReleaseAttributeContext(DataContext); ExFreePoolWithTag(BitmapData, TAG_NTFS); ExFreePoolWithTag(BitmapRecord, TAG_NTFS); + + // Save updated data runs to file record + + // Allocate some memory for a new RunBuffer + RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); + if (!RunBuffer) + { + DPRINT1("ERROR: Couldn't allocate memory for data runs!\n"); + return STATUS_INSUFFICIENT_RESOURCES; + }
// Convert the map control block back to encoded data runs - ConvertLargeMCBToDataRuns(&DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferOffset); + ConvertLargeMCBToDataRuns(&AttrContext->DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferSize);
// Update HighestVCN DestinationAttribute->NonResident.HighestVCN = AttrContext->Record.NonResident.HighestVCN; @@ -836,27 +797,23 @@ // Write data runs to destination attribute RtlCopyMemory((PVOID)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), RunBuffer, - RunBufferOffset); - + RunBufferSize); + + // Is DestinationAttribute the last attribute in the file record? if (NextAttribute->Type == AttributeEnd) { // update attribute length - AttrContext->Record.Length = ALIGN_UP_BY(AttrContext->Record.NonResident.MappingPairsOffset + RunBufferOffset, 8); + AttrContext->Record.Length = ALIGN_UP_BY(AttrContext->Record.NonResident.MappingPairsOffset + RunBufferSize, 8); DestinationAttribute->Length = AttrContext->Record.Length;
// write end markers NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)DestinationAttribute + DestinationAttribute->Length); - NextAttribute->Type = AttributeEnd; - NextAttribute->Length = FILE_RECORD_END; - - // update file record length - FileRecord->BytesInUse = AttrOffset + DestinationAttribute->Length + (sizeof(ULONG) * 2); + SetFileRecordEnd(FileRecord, NextAttribute, FILE_RECORD_END); }
// Update the file record Status = UpdateFileRecord(Vcb, AttrContext->FileMFTIndex, FileRecord);
- FsRtlUninitializeLargeMcb(&DataRunsMCB); ExFreePoolWithTag(RunBuffer, TAG_NTFS);
NtfsDumpDataRuns((PUCHAR)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), 0);
Modified: branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/mft.c URL: http://svn.reactos.org/svn/reactos/branches/GSoC_2016/NTFS/drivers/filesyste... ============================================================================== --- branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/mft.c [iso-8859-1] (original) +++ branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/mft.c [iso-8859-1] Fri May 12 22:16:20 2017 @@ -50,8 +50,10 @@ { LONGLONG DataRunOffset; ULONGLONG DataRunLength; - - Context->CacheRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset; + ULONGLONG NextVBN = 0; + PUCHAR DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset; + + Context->CacheRun = DataRun; Context->CacheRunOffset = 0; Context->CacheRun = DecodeRun(Context->CacheRun, &DataRunOffset, &DataRunLength); Context->CacheRunLength = DataRunLength; @@ -68,6 +70,14 @@ Context->CacheRunLastLCN = 0; } Context->CacheRunCurrentOffset = 0; + + // Convert the data runs to a map control block + if (!NT_SUCCESS(ConvertDataRunsToLargeMCB(DataRun, &Context->DataRunsMCB, &NextVBN))) + { + DPRINT1("Unable to convert data runs to MCB!\n"); + ExFreePoolWithTag(Context, TAG_NTFS); + return NULL; + } }
return Context; @@ -77,6 +87,11 @@ VOID ReleaseAttributeContext(PNTFS_ATTR_CONTEXT Context) { + if (Context->Record.IsNonResident) + { + FsRtlUninitializeLargeMcb(&Context->DataRunsMCB); + } + ExFreePoolWithTag(Context, TAG_NTFS); }
@@ -246,10 +261,30 @@ ULONG NextAssignedCluster; ULONG AssignedClusters;
- NTSTATUS Status = GetLastClusterInDataRun(Fcb->Vcb, &AttrContext->Record, (PULONGLONG)&LastClusterInDataRun.QuadPart); - - DPRINT1("GetLastClusterInDataRun returned: %I64u\n", LastClusterInDataRun.QuadPart); - DPRINT1("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN); + if (ExistingClusters == 0) + { + LastClusterInDataRun.QuadPart = 0; + } + else + { + if (!FsRtlLookupLargeMcbEntry(&AttrContext->DataRunsMCB, + (LONGLONG)AttrContext->Record.NonResident.HighestVCN, + (PLONGLONG)&LastClusterInDataRun.QuadPart, + NULL, + NULL, + NULL, + NULL)) + { + DPRINT1("Error looking up final large MCB entry!\n"); + + // Most likely, HighestVCN went above the largest mapping + DPRINT1("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN); + return STATUS_INVALID_PARAMETER; + } + } + + DPRINT("LastClusterInDataRun: %I64u\n", LastClusterInDataRun.QuadPart); + DPRINT("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN);
while (ClustersNeeded > 0) { @@ -405,6 +440,9 @@ ULONG ReadLength; ULONG AlreadyRead; NTSTATUS Status; + + //TEMPTEMP + PUCHAR TempBuffer;
if (!Context->Record.IsNonResident) { @@ -438,9 +476,20 @@ } else { + //TEMPTEMP + ULONG UsedBufferSize; + TempBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); + LastLCN = 0; - DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset; CurrentOffset = 0; + + // This will be rewritten in the next iteration to just use the DataRuns MCB directly + ConvertLargeMCBToDataRuns(&Context->DataRunsMCB, + TempBuffer, + Vcb->NtfsInfo.BytesPerFileRecord, + &UsedBufferSize); + + DataRun = TempBuffer;
while (1) { @@ -557,6 +606,10 @@ } /* while */
} /* if Disk */ + + // TEMPTEMP + if (Context->Record.IsNonResident) + ExFreePoolWithTag(TempBuffer, TAG_NTFS);
Context->CacheRun = DataRun; Context->CacheRunOffset = Offset + AlreadyRead; @@ -622,6 +675,10 @@ NTSTATUS Status; PUCHAR SourceBuffer = Buffer; LONGLONG StartingOffset; + + //TEMPTEMP + PUCHAR TempBuffer; +
DPRINT("WriteAttribute(%p, %p, %I64u, %p, %lu, %p)\n", Vcb, Context, Offset, Buffer, Length, RealLengthWritten);
@@ -707,9 +764,19 @@ } else*/ { + ULONG UsedBufferSize; LastLCN = 0; - DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset; - CurrentOffset = 0; + CurrentOffset = 0; + + // This will be rewritten in the next iteration to just use the DataRuns MCB directly + TempBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS); + + ConvertLargeMCBToDataRuns(&Context->DataRunsMCB, + TempBuffer, + Vcb->NtfsInfo.BytesPerFileRecord, + &UsedBufferSize); + + DataRun = TempBuffer;
while (1) { @@ -863,6 +930,10 @@ DataRunStartLCN = -1; } } // end while (Length > 0) [more data to write] + + // TEMPTEMP + if(Context->Record.IsNonResident) + ExFreePoolWithTag(TempBuffer, TAG_NTFS);
Context->CacheRun = DataRun; Context->CacheRunOffset = Offset + *RealLengthWritten;
Modified: branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/ntfs.h URL: http://svn.reactos.org/svn/reactos/branches/GSoC_2016/NTFS/drivers/filesyste... ============================================================================== --- branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/ntfs.h [iso-8859-1] (original) +++ branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/ntfs.h [iso-8859-1] Fri May 12 22:16:20 2017 @@ -437,6 +437,7 @@ ULONGLONG CacheRunLength; LONGLONG CacheRunLastLCN; ULONGLONG CacheRunCurrentOffset; + LARGE_MCB DataRunsMCB; ULONGLONG FileMFTIndex; NTFS_ATTR_RECORD Record; } NTFS_ATTR_CONTEXT, *PNTFS_ATTR_CONTEXT;
Modified: branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/rw.c URL: http://svn.reactos.org/svn/reactos/branches/GSoC_2016/NTFS/drivers/filesyste... ============================================================================== --- branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/rw.c [iso-8859-1] (original) +++ branches/GSoC_2016/NTFS/drivers/filesystems/ntfs/rw.c [iso-8859-1] Fri May 12 22:16:20 2017 @@ -432,22 +432,6 @@ return Status; }
- // at this point the record in DataContext may be stale, so we need to refresh it - ReleaseAttributeContext(DataContext); - - Status = FindAttribute(DeviceExt, - FileRecord, - AttributeData, - Fcb->Stream, - wcslen(Fcb->Stream), - &DataContext, - &AttributeOffset); - if (!NT_SUCCESS(Status)) - { - DPRINT1("DRIVER ERROR: Couldn't find $DATA attribute after setting size!\n"); - return Status; - } - // now we need to update this file's size in every directory index entry that references it // TODO: put this code in its own function and adapt it to work with every filename / hardlink // stored in the file record.