To create a virtual file system / registry, BoxedApp SDK (this, by the way, is exactly what applications created with BoxedApp Packer use) uses the interception of system functions technique. A number of original ideas have allowed us to create an interception system compatible with any environment, and now the part of SDK that is in charge of the interception has become accessible to developers - SDK users.

Here's an example of how to disable creating and opening the file named "1.txt":

typedef HANDLE (WINAPI *P_CreateFileW)(
	LPCWSTR lpFileName,
	DWORD dwDesiredAccess,
	DWORD dwShareMode,
	LPSECURITY_ATTRIBUTES lpSecurityAttributes,
	DWORD dwCreationDisposition,
	DWORD dwFlagsAndAttributes,
	HANDLE hTemplateFile);
P_CreateFileW g_pCreateFileW;

HANDLE WINAPI My_CreateFileW(
	LPCWSTR lpFileName,
	DWORD dwDesiredAccess,
	DWORD dwShareMode,
	LPSECURITY_ATTRIBUTES lpSecurityAttributes,
	DWORD dwCreationDisposition,
	DWORD dwFlagsAndAttributes,
	HANDLE hTemplateFile)
{
	if (0 == lstrcmpiW(lpFileName, L"1.txt"))
	{
		SetLastError(ERROR_FILE_EXISTS);
		return INVALID_HANDLE_VALUE;
	}
	else
		return g_pCreateFileW(
				lpFileName,
				dwDesiredAccess,
				dwShareMode,
				lpSecurityAttributes,
				dwCreationDisposition,
				dwFlagsAndAttributes,
				hTemplateFile);
}
...
BoxedAppSDK_Init();

PVOID pCreateFileW = GetProcAddress(GetModuleHandle(_T("kernel32.dll")), "CreateFileW");

HANDLE hHook__CreateFileW = BoxedAppSDK_HookFunction(pCreateFileW, &My_CreateFileW, TRUE);

g_pCreateFileW = (P_CreateFileW)BoxedAppSDK_GetOriginalFunction(hHook__CreateFileW);

FILE* f = fopen("1.txt", "r");

// f is NULL
...

BoxedAppSDK_UnhookFunction(hHook__CreateFileW);
			

Delphi version of the same idea:

type
TCreateFileW = 
   function(lpFileName: PWideChar;
            dwDesiredAccess, dwShareMode: Integer;
            lpSecurityAttributes: PSecurityAttributes;
            dwCreationDisposition, dwFlagsAndAttributes: DWORD;
            hTemplateFile: THandle): THandle; stdcall;

var
   OriginalCreateFileW: TCreateFileW;

function My_CreateFileW(
         lpFileName: PWideChar;
         dwDesiredAccess, dwShareMode: Integer;
         lpSecurityAttributes: PSecurityAttributes;
         dwCreationDisposition, dwFlagsAndAttributes: DWORD;
         hTemplateFile: THandle): THandle; stdcall;
begin
   if 0 = lstrcmpiW(lpFileName, '1.txt') then
   begin
      Result := INVALID_HANDLE_VALUE;
      SetLastError(ERROR_ALREADY_EXISTS);
   end
   else
      Result := 
        OriginalCreateFileW(
           lpFileName, 
           dwDesiredAccess,
           dwShareMode, 
           lpSecurityAttributes, 
           dwCreationDisposition, 
           dwFlagsAndAttributes, 
           hTemplateFile);
end;

var
   pCreateFileW: Pointer;
   hHook__CreateFileW: THandle;

begin
  Application.Initialize;

  BoxedAppSDK_Init;

  pCreateFileW := GetProcAddress(GetModuleHandle('kernel32.dll'), 'CreateFileW');
  hHook__CreateFileW := BoxedAppSDK_HookFunction(pCreateFileW, @My_CreateFileW, TRUE);
  OriginalCreateFileW := BoxedAppSDK_GetOriginalFunction(hHook__CreateFileW);

  // This line produces an exception because we prevent creating / opening '1.txt'
  TFileStream.Create('1.txt', fmCreate or fmOpenRead);

  BoxedAppSDK_UnhookFunction(hHook__CreateFileW);
end.