/**************************************************************************** * * The MIT License (MIT) * * Copyright (c) 2014 Vivante Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * ***************************************************************************** * * The GPL License (GPL) * * Copyright (C) 2014 Vivante Corporation * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************** * * Note: This software is released under dual MIT and GPL licenses. A * recipient may use this file under the terms of either the MIT license or * GPL License. If you wish to use only one license not the other, you can * indicate your decision by deleting one of the above license notices in your * version of this file. * *****************************************************************************/ #include "gc_hal_kernel_precomp.h" #define _GC_OBJ_ZONE gcvZONE_MMU typedef enum _gceMMU_TYPE { gcvMMU_USED = (0 << 4), gcvMMU_SINGLE = (1 << 4), gcvMMU_FREE = (2 << 4), } gceMMU_TYPE; #define gcmENTRY_TYPE(x) (x & 0xF0) #define gcdMMU_TABLE_DUMP 0 #define gcdUSE_MMU_EXCEPTION 0 /* gcdMMU_CLEAR_VALUE The clear value for the entry of the old MMU. */ #ifndef gcdMMU_CLEAR_VALUE # define gcdMMU_CLEAR_VALUE 0x00000ABC #endif #define gcdVERTEX_START (128 << 10) typedef struct _gcsMMU_STLB *gcsMMU_STLB_PTR; typedef struct _gcsMMU_STLB { gctPHYS_ADDR physical; gctUINT32_PTR logical; gctSIZE_T size; gctUINT32 physBase; gctSIZE_T pageCount; gctUINT32 mtlbIndex; gctUINT32 mtlbEntryNum; gcsMMU_STLB_PTR next; } gcsMMU_STLB; #if gcdSHARED_PAGETABLE typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR; typedef struct _gcsSharedPageTable { /* Shared gckMMU object. */ gckMMU mmu; /* Hardwares which use this shared pagetable. */ gckHARDWARE hardwares[gcdMAX_GPU_COUNT]; /* Number of cores use this shared pagetable. */ gctUINT32 reference; } gcsSharedPageTable; static gcsSharedPageTable_PTR sharedPageTable = gcvNULL; #endif #if gcdMIRROR_PAGETABLE typedef struct _gcsMirrorPageTable * gcsMirrorPageTable_PTR; typedef struct _gcsMirrorPageTable { /* gckMMU objects. */ gckMMU mmus[gcdMAX_GPU_COUNT]; /* Hardwares which use this shared pagetable. */ gckHARDWARE hardwares[gcdMAX_GPU_COUNT]; /* Number of cores use this shared pagetable. */ gctUINT32 reference; } gcsMirrorPageTable; static gcsMirrorPageTable_PTR mirrorPageTable = gcvNULL; static gctPOINTER mirrorPageTableMutex = gcvNULL; #endif typedef struct _gcsDynamicSpaceNode * gcsDynamicSpaceNode_PTR; typedef struct _gcsDynamicSpaceNode { gctUINT32 start; gctINT32 entries; } gcsDynamicSpaceNode; static void _WritePageEntry( IN gctUINT32_PTR PageEntry, IN gctUINT32 EntryValue ) { static gctUINT16 data = 0xff00; if (*(gctUINT8 *)&data == 0xff) { *PageEntry = gcmSWAB32(EntryValue); } else { *PageEntry = EntryValue; } } static gctUINT32 _ReadPageEntry( IN gctUINT32_PTR PageEntry ) { static gctUINT16 data = 0xff00; gctUINT32 entryValue; if (*(gctUINT8 *)&data == 0xff) { entryValue = *PageEntry; return gcmSWAB32(entryValue); } else { return *PageEntry; } } static gceSTATUS _FillPageTable( IN gctUINT32_PTR PageTable, IN gctUINT32 PageCount, IN gctUINT32 EntryValue ) { gctUINT i; for (i = 0; i < PageCount; i++) { _WritePageEntry(PageTable + i, EntryValue); } return gcvSTATUS_OK; } static gceSTATUS _Link( IN gckMMU Mmu, IN gctUINT32 Index, IN gctUINT32 Next ) { if (Index >= Mmu->pageTableEntries) { /* Just move heap pointer. */ Mmu->heapList = Next; } else { /* Address page table. */ gctUINT32_PTR map = Mmu->mapLogical; /* Dispatch on node type. */ switch (gcmENTRY_TYPE(_ReadPageEntry(&map[Index]))) { case gcvMMU_SINGLE: /* Set single index. */ _WritePageEntry(&map[Index], (Next << 8) | gcvMMU_SINGLE); break; case gcvMMU_FREE: /* Set index. */ _WritePageEntry(&map[Index + 1], Next); break; default: gcmkFATAL("MMU table correcupted at index %u!", Index); return gcvSTATUS_HEAP_CORRUPTED; } } /* Success. */ return gcvSTATUS_OK; } static gceSTATUS _AddFree( IN gckMMU Mmu, IN gctUINT32 Index, IN gctUINT32 Node, IN gctUINT32 Count ) { gctUINT32_PTR map = Mmu->mapLogical; if (Count == 1) { /* Initialize a single page node. */ _WritePageEntry(map + Node, (~((1U<<8)-1)) | gcvMMU_SINGLE); } else { /* Initialize the node. */ _WritePageEntry(map + Node + 0, (Count << 8) | gcvMMU_FREE); _WritePageEntry(map + Node + 1, ~0U); } /* Append the node. */ return _Link(Mmu, Index, Node); } static gceSTATUS _Collect( IN gckMMU Mmu ) { gctUINT32_PTR map = Mmu->mapLogical; gceSTATUS status; gctUINT32 i, previous, start = 0, count = 0; previous = Mmu->heapList = ~0U; Mmu->freeNodes = gcvFALSE; /* Walk the entire page table. */ for (i = 0; i < Mmu->pageTableEntries; ++i) { /* Dispatch based on type of page. */ switch (gcmENTRY_TYPE(_ReadPageEntry(&map[i]))) { case gcvMMU_USED: /* Used page, so close any open node. */ if (count > 0) { /* Add the node. */ gcmkONERROR(_AddFree(Mmu, previous, start, count)); /* Reset the node. */ previous = start; count = 0; } break; case gcvMMU_SINGLE: /* Single free node. */ if (count++ == 0) { /* Start a new node. */ start = i; } break; case gcvMMU_FREE: /* A free node. */ if (count == 0) { /* Start a new node. */ start = i; } /* Advance the count. */ count += _ReadPageEntry(&map[i]) >> 8; /* Advance the index into the page table. */ i += (_ReadPageEntry(&map[i]) >> 8) - 1; break; default: gcmkFATAL("MMU page table correcupted at index %u!", i); return gcvSTATUS_HEAP_CORRUPTED; } } /* See if we have an open node left. */ if (count > 0) { /* Add the node to the list. */ gcmkONERROR(_AddFree(Mmu, previous, start, count)); } gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU, "Performed a garbage collection of the MMU heap."); /* Success. */ return gcvSTATUS_OK; OnError: /* Return the staus. */ return status; } static gctUINT32 _SetPage(gctUINT32 PageAddress, gctUINT32 PageAddressExt) { return PageAddress /* AddressExt */ | (PageAddressExt << 4) /* writable */ | (1 << 2) /* Ignore exception */ | (0 << 1) /* Present */ | (1 << 0); } #if gcdPROCESS_ADDRESS_SPACE gctUINT32 _AddressToIndex( IN gckMMU Mmu, IN gctUINT32 Address ) { gctUINT32 mtlbOffset = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; gctUINT32 stlbOffset = (Address & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT; return (mtlbOffset - Mmu->dynamicMappingStart) * gcdMMU_STLB_4K_ENTRY_NUM + stlbOffset; } gctUINT32 _MtlbOffset( gctUINT32 Address ) { return (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; } gctUINT32 _StlbOffset( gctUINT32 Address ) { return (Address & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT; } static gceSTATUS _AllocateStlb( IN gckOS Os, OUT gcsMMU_STLB_PTR *Stlb ) { gceSTATUS status; gcsMMU_STLB_PTR stlb; gctPOINTER pointer; /* Allocate slave TLB record. */ gcmkONERROR(gckOS_Allocate(Os, gcmSIZEOF(gcsMMU_STLB), &pointer)); stlb = pointer; stlb->size = gcdMMU_STLB_4K_SIZE; /* Allocate slave TLB entries. */ gcmkONERROR(gckOS_AllocateContiguous( Os, gcvFALSE, &stlb->size, &stlb->physical, (gctPOINTER)&stlb->logical )); gcmkONERROR(gckOS_GetPhysicalAddress(Os, stlb->logical, &stlb->physBase)); #if gcdUSE_MMU_EXCEPTION _FillPageTable(stlb->logical, stlb->size / 4, gcdMMU_STLB_EXCEPTION); #else gckOS_ZeroMemory(stlb->logical, stlb->size); #endif *Stlb = stlb; return gcvSTATUS_OK; OnError: return status; } gceSTATUS _SetupProcessAddressSpace( IN gckMMU Mmu ) { gceSTATUS status; gctINT numEntries = 0; gctUINT32_PTR map; numEntries = gcdPROCESS_ADDRESS_SPACE_SIZE /* Address space mapped by one MTLB entry. */ / (1 << gcdMMU_MTLB_SHIFT); Mmu->dynamicMappingStart = 0; Mmu->pageTableSize = numEntries * 4096; Mmu->pageTableEntries = Mmu->pageTableSize / gcmSIZEOF(gctUINT32); gcmkONERROR(gckOS_Allocate(Mmu->os, Mmu->pageTableSize, (void **)&Mmu->mapLogical)); /* Initilization. */ map = Mmu->mapLogical; _WritePageEntry(map, (Mmu->pageTableEntries << 8) | gcvMMU_FREE); _WritePageEntry(map + 1, ~0U); Mmu->heapList = 0; Mmu->freeNodes = gcvFALSE; return gcvSTATUS_OK; OnError: return status; } #else static gceSTATUS _FillFlatMapping( IN gckMMU Mmu, IN gctUINT32 PhysBase, OUT gctSIZE_T Size ) { gceSTATUS status; gctBOOL mutex = gcvFALSE; gcsMMU_STLB_PTR head = gcvNULL, pre = gcvNULL; gctUINT32 start = PhysBase & (~gcdMMU_PAGE_64K_MASK); gctUINT32 end = (PhysBase + Size - 1) & (~gcdMMU_PAGE_64K_MASK); gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT; gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT; gctUINT32 sStart = (start & gcdMMU_STLB_64K_MASK) >> gcdMMU_STLB_64K_SHIFT; gctUINT32 sEnd = (end & gcdMMU_STLB_64K_MASK) >> gcdMMU_STLB_64K_SHIFT; gctBOOL ace = gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_ACE); gctPHYS_ADDR_T physical; /* Grab the mutex. */ gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); mutex = gcvTRUE; while (mStart <= mEnd) { gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM); if (*(Mmu->mtlbLogical + mStart) == 0) { gcsMMU_STLB_PTR stlb; gctPOINTER pointer = gcvNULL; gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_64K_ENTRY_NUM - 1); gctUINT32 mtlbEntry; gcmkONERROR(gckOS_Allocate(Mmu->os, sizeof(struct _gcsMMU_STLB), &pointer)); stlb = pointer; stlb->mtlbEntryNum = 0; stlb->next = gcvNULL; stlb->physical = gcvNULL; stlb->logical = gcvNULL; stlb->size = gcdMMU_STLB_64K_SIZE; stlb->pageCount = 0; if (pre == gcvNULL) { pre = head = stlb; } else { gcmkASSERT(pre->next == gcvNULL); pre->next = stlb; pre = stlb; } gcmkONERROR( gckOS_AllocateContiguous(Mmu->os, gcvFALSE, &stlb->size, &stlb->physical, (gctPOINTER)&stlb->logical)); gcmkONERROR(gckOS_ZeroMemory(stlb->logical, stlb->size)); gcmkONERROR(gckOS_GetPhysicalAddress( Mmu->os, stlb->logical, &physical)); gcmkSAFECASTPHYSADDRT(stlb->physBase, physical); if (stlb->physBase & (gcdMMU_STLB_64K_SIZE - 1)) { gcmkONERROR(gcvSTATUS_NOT_ALIGNED); } mtlbEntry = stlb->physBase /* 64KB page size */ | (1 << 2) /* Ignore exception */ | (0 << 1) /* Present */ | (1 << 0); if (ace) { mtlbEntry = mtlbEntry /* Secure */ | (1 << 4); } _WritePageEntry(Mmu->mtlbLogical + mStart, mtlbEntry); #if gcdMMU_TABLE_DUMP gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", __FUNCTION__, __LINE__, mStart, _ReadPageEntry(Mmu->mtlbLogical + mStart)); #endif stlb->mtlbIndex = mStart; stlb->mtlbEntryNum = 1; #if gcdMMU_TABLE_DUMP gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n", __FUNCTION__, __LINE__, stlb->logical, stlb->physBase); #endif while (sStart <= last) { gcmkASSERT(!(start & gcdMMU_PAGE_64K_MASK)); _WritePageEntry(stlb->logical + sStart, _SetPage(start, 0)); #if gcdMMU_TABLE_DUMP gckOS_Print("%s(%d): insert STLB[%d]: %08x\n", __FUNCTION__, __LINE__, sStart, _ReadPageEntry(stlb->logical + sStart)); #endif /* next page. */ start += gcdMMU_PAGE_64K_SIZE; sStart++; stlb->pageCount++; } sStart = 0; ++mStart; } else { gcmkONERROR(gcvSTATUS_INVALID_REQUEST); } } /* Insert the stlb into staticSTLB. */ if (Mmu->staticSTLB == gcvNULL) { Mmu->staticSTLB = head; } else { gcmkASSERT(pre == gcvNULL); gcmkASSERT(pre->next == gcvNULL); pre->next = Mmu->staticSTLB; Mmu->staticSTLB = head; } /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); return gcvSTATUS_OK; OnError: /* Roll back. */ while (head != gcvNULL) { pre = head; head = head->next; if (pre->physical != gcvNULL) { gcmkVERIFY_OK( gckOS_FreeContiguous(Mmu->os, pre->physical, pre->logical, pre->size)); } if (pre->mtlbEntryNum != 0) { gcmkASSERT(pre->mtlbEntryNum == 1); _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex, 0); } gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre)); } if (mutex) { /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); } return status; } static gceSTATUS _FindDynamicSpace( IN gckMMU Mmu, OUT gcsDynamicSpaceNode_PTR *Array, OUT gctINT * Size ) { gceSTATUS status = gcvSTATUS_OK; gctPOINTER pointer = gcvNULL; gcsDynamicSpaceNode_PTR array = gcvNULL; gctINT size = 0; gctINT i = 0, nodeStart = -1, nodeEntries = 0; /* Allocate memory for the array. */ gcmkONERROR(gckOS_Allocate(Mmu->os, gcmSIZEOF(*array) * (gcdMMU_MTLB_ENTRY_NUM / 2), &pointer)); array = (gcsDynamicSpaceNode_PTR)pointer; /* Loop all the entries. */ while (i < gcdMMU_MTLB_ENTRY_NUM) { if (!Mmu->mtlbLogical[i]) { if (nodeStart < 0) { /* This is the first entry of the dynamic space. */ nodeStart = i; nodeEntries = 1; } else { /* Other entries of the dynamic space. */ nodeEntries++; } } else if (nodeStart >= 0) { /* Save the previous node. */ array[size].start = nodeStart; array[size].entries = nodeEntries; size++; /* Reset the start. */ nodeStart = -1; nodeEntries = 0; } i++; } /* Save the previous node. */ if (nodeStart >= 0) { array[size].start = nodeStart; array[size].entries = nodeEntries; size++; } #if gcdMMU_TABLE_DUMP for (i = 0; i < size; i++) { gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n", __FUNCTION__, __LINE__, i, array[i].start, array[i].entries); } #endif *Array = array; *Size = size; return gcvSTATUS_OK; OnError: if (pointer != gcvNULL) { gckOS_Free(Mmu->os, pointer); } return status; } static gceSTATUS _SetupDynamicSpace( IN gckMMU Mmu ) { gceSTATUS status; gcsDynamicSpaceNode_PTR nodeArray = gcvNULL; gctINT i, nodeArraySize = 0; gctPHYS_ADDR_T physical; gctUINT32 address; gctINT numEntries = 0; gctUINT32_PTR map; gctBOOL acquired = gcvFALSE; gctUINT32 mtlbEntry; gctBOOL ace = gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_ACE); /* Find all the dynamic address space. */ gcmkONERROR(_FindDynamicSpace(Mmu, &nodeArray, &nodeArraySize)); /* TODO: We only use the largest one for now. */ for (i = 0; i < nodeArraySize; i++) { if (nodeArray[i].entries > numEntries) { Mmu->dynamicMappingStart = nodeArray[i].start; numEntries = nodeArray[i].entries; } } gckOS_Free(Mmu->os, (gctPOINTER)nodeArray); Mmu->pageTableSize = numEntries * 4096; gcmkSAFECASTSIZET(Mmu->pageTableEntries, Mmu->pageTableSize / gcmSIZEOF(gctUINT32)); gcmkONERROR(gckOS_Allocate(Mmu->os, Mmu->pageTableSize, (void **)&Mmu->mapLogical)); /* Construct Slave TLB. */ gcmkONERROR(gckOS_AllocateContiguous(Mmu->os, gcvFALSE, &Mmu->pageTableSize, &Mmu->pageTablePhysical, (gctPOINTER)&Mmu->pageTableLogical)); #if gcdUSE_MMU_EXCEPTION gcmkONERROR(_FillPageTable(Mmu->pageTableLogical, Mmu->pageTableEntries, /* Enable exception */ 1 << 1)); #else /* Invalidate all entries. */ gcmkONERROR(gckOS_ZeroMemory(Mmu->pageTableLogical, Mmu->pageTableSize)); #endif /* Initilization. */ map = Mmu->mapLogical; _WritePageEntry(map, (Mmu->pageTableEntries << 8) | gcvMMU_FREE); _WritePageEntry(map + 1, ~0U); Mmu->heapList = 0; Mmu->freeNodes = gcvFALSE; gcmkONERROR(gckOS_GetPhysicalAddress(Mmu->os, Mmu->pageTableLogical, &physical)); gcmkSAFECASTPHYSADDRT(address, physical); /* Grab the mutex. */ gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); acquired = gcvTRUE; /* Map to Master TLB. */ for (i = (gctINT)Mmu->dynamicMappingStart; i < (gctINT)Mmu->dynamicMappingStart + numEntries; i++) { mtlbEntry = address /* 4KB page size */ | (0 << 2) /* Ignore exception */ | (0 << 1) /* Present */ | (1 << 0); if (ace) { mtlbEntry = mtlbEntry /* Secure */ | (1 << 4); } _WritePageEntry(Mmu->mtlbLogical + i, mtlbEntry); #if gcdMMU_TABLE_DUMP gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n", __FUNCTION__, __LINE__, i, _ReadPageEntry(Mmu->mtlbLogical + i)); #endif address += gcdMMU_STLB_4K_SIZE; } /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); return gcvSTATUS_OK; OnError: if (Mmu->mapLogical) { gcmkVERIFY_OK( gckOS_Free(Mmu->os, (gctPOINTER) Mmu->mapLogical)); gcmkVERIFY_OK( gckOS_FreeContiguous(Mmu->os, Mmu->pageTablePhysical, (gctPOINTER) Mmu->pageTableLogical, Mmu->pageTableSize)); } if (acquired) { /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); } return status; } #endif /******************************************************************************* ** ** _Construct ** ** Construct a new gckMMU object. ** ** INPUT: ** ** gckKERNEL Kernel ** Pointer to an gckKERNEL object. ** ** gctSIZE_T MmuSize ** Number of bytes for the page table. ** ** OUTPUT: ** ** gckMMU * Mmu ** Pointer to a variable that receives the gckMMU object pointer. */ gceSTATUS _Construct( IN gckKERNEL Kernel, IN gctSIZE_T MmuSize, OUT gckMMU * Mmu ) { gckOS os; gckHARDWARE hardware; gceSTATUS status; gckMMU mmu = gcvNULL; gctUINT32_PTR map; gctPOINTER pointer = gcvNULL; #if gcdPROCESS_ADDRESS_SPACE gctUINT32 i; gctUINT32 physical; #endif gctUINT32 physBase; gctUINT32 physSize; gctUINT32 gpuAddress; gctPHYS_ADDR_T gpuPhysical; gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL); gcmkVERIFY_ARGUMENT(MmuSize > 0); gcmkVERIFY_ARGUMENT(Mmu != gcvNULL); /* Extract the gckOS object pointer. */ os = Kernel->os; gcmkVERIFY_OBJECT(os, gcvOBJ_OS); /* Extract the gckHARDWARE object pointer. */ hardware = Kernel->hardware; gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE); /* Allocate memory for the gckMMU object. */ gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer)); mmu = pointer; /* Initialize the gckMMU object. */ mmu->object.type = gcvOBJ_MMU; mmu->os = os; mmu->hardware = hardware; mmu->pageTableMutex = gcvNULL; mmu->pageTableLogical = gcvNULL; mmu->mtlbLogical = gcvNULL; mmu->staticSTLB = gcvNULL; mmu->enabled = gcvFALSE; mmu->mapLogical = gcvNULL; /* Create the page table mutex. */ gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex)); if (hardware->mmuVersion == 0) { mmu->pageTableSize = MmuSize; /* Construct address space management table. */ gcmkONERROR(gckOS_Allocate(mmu->os, mmu->pageTableSize, &pointer)); mmu->mapLogical = pointer; /* Construct page table read by GPU. */ gcmkONERROR(gckOS_AllocateContiguous(mmu->os, gcvFALSE, &mmu->pageTableSize, &mmu->pageTablePhysical, (gctPOINTER)&mmu->pageTableLogical)); /* Compute number of entries in page table. */ gcmkSAFECASTSIZET(mmu->pageTableEntries, mmu->pageTableSize / sizeof(gctUINT32)); /* Mark all pages as free. */ map = mmu->mapLogical; #if gcdMMU_CLEAR_VALUE _FillPageTable(mmu->pageTableLogical, mmu->pageTableEntries, gcdMMU_CLEAR_VALUE); #endif _WritePageEntry(map, (mmu->pageTableEntries << 8) | gcvMMU_FREE); _WritePageEntry(map + 1, ~0U); mmu->heapList = 0; mmu->freeNodes = gcvFALSE; } else { /* Allocate the 4K mode MTLB table. */ mmu->mtlbSize = gcdMMU_MTLB_SIZE + 64; gcmkONERROR( gckOS_AllocateContiguous(os, gcvFALSE, &mmu->mtlbSize, &mmu->mtlbPhysical, &pointer)); mmu->mtlbLogical = pointer; #if gcdPROCESS_ADDRESS_SPACE _FillPageTable(pointer, mmu->mtlbSize / 4, gcdMMU_MTLB_EXCEPTION); /* Allocate a array to store stlbs. */ gcmkONERROR(gckOS_Allocate(os, mmu->mtlbSize, &mmu->stlbs)); gckOS_ZeroMemory(mmu->stlbs, mmu->mtlbSize); for (i = 0; i < gcdMAX_GPU_COUNT; i++) { gcmkONERROR(gckOS_AtomConstruct(os, &mmu->pageTableDirty[i])); } _SetupProcessAddressSpace(mmu); /* Map kernel command buffer in MMU. */ for (i = 0; i < gcdCOMMAND_QUEUES; i++) { gcmkONERROR(gckOS_GetPhysicalAddress( mmu->os, Kernel->command->queues[i].logical, &physical )); gcmkONERROR(gckMMU_FlatMapping(mmu, physical)); } #else /* Invalid all the entries. */ gcmkONERROR( gckOS_ZeroMemory(pointer, mmu->mtlbSize)); gcmkONERROR( gckOS_QueryOption(mmu->os, "physBase", &physBase)); gcmkONERROR( gckOS_QueryOption(mmu->os, "physSize", &physSize)); gcmkONERROR( gckOS_CPUPhysicalToGPUPhysical(mmu->os, physBase, &gpuPhysical)); gcmkSAFECASTPHYSADDRT(gpuAddress, gpuPhysical); /* Setup [physBase - physSize) flat mapping. */ gcmkONERROR(_FillFlatMapping( mmu, gpuAddress, physSize )); gcmkONERROR(_SetupDynamicSpace(mmu)); #endif } /* Return the gckMMU object pointer. */ *Mmu = mmu; /* Success. */ gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu); return gcvSTATUS_OK; OnError: /* Roll back. */ if (mmu != gcvNULL) { if (mmu->mapLogical != gcvNULL) { gcmkVERIFY_OK( gckOS_Free(os, (gctPOINTER) mmu->mapLogical)); gcmkVERIFY_OK( gckOS_FreeContiguous(os, mmu->pageTablePhysical, (gctPOINTER) mmu->pageTableLogical, mmu->pageTableSize)); } if (mmu->mtlbLogical != gcvNULL) { gcmkVERIFY_OK( gckOS_FreeContiguous(os, mmu->mtlbPhysical, (gctPOINTER) mmu->mtlbLogical, mmu->mtlbSize)); } if (mmu->pageTableMutex != gcvNULL) { /* Delete the mutex. */ gcmkVERIFY_OK( gckOS_DeleteMutex(os, mmu->pageTableMutex)); } /* Mark the gckMMU object as unknown. */ mmu->object.type = gcvOBJ_UNKNOWN; /* Free the allocates memory. */ gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu)); } /* Return the status. */ gcmkFOOTER(); return status; } /******************************************************************************* ** ** _Destroy ** ** Destroy a gckMMU object. ** ** INPUT: ** ** gckMMU Mmu ** Pointer to an gckMMU object. ** ** OUTPUT: ** ** Nothing. */ gceSTATUS _Destroy( IN gckMMU Mmu ) { #if gcdPROCESS_ADDRESS_SPACE gctUINT32 i; #endif gcmkHEADER_ARG("Mmu=0x%x", Mmu); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); while (Mmu->staticSTLB != gcvNULL) { gcsMMU_STLB_PTR pre = Mmu->staticSTLB; Mmu->staticSTLB = pre->next; if (pre->physical != gcvNULL) { gcmkVERIFY_OK( gckOS_FreeContiguous(Mmu->os, pre->physical, pre->logical, pre->size)); } if (pre->mtlbEntryNum != 0) { gcmkASSERT(pre->mtlbEntryNum == 1); _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex, 0); #if gcdMMU_TABLE_DUMP gckOS_Print("%s(%d): clean MTLB[%d]\n", __FUNCTION__, __LINE__, pre->mtlbIndex); #endif } gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre)); } if (Mmu->hardware->mmuVersion != 0) { gcmkVERIFY_OK( gckOS_FreeContiguous(Mmu->os, Mmu->mtlbPhysical, (gctPOINTER) Mmu->mtlbLogical, Mmu->mtlbSize)); } /* Free address space management table. */ if (Mmu->mapLogical != gcvNULL) { gcmkVERIFY_OK( gckOS_Free(Mmu->os, (gctPOINTER) Mmu->mapLogical)); } if (Mmu->pageTableLogical != gcvNULL) { /* Free page table. */ gcmkVERIFY_OK( gckOS_FreeContiguous(Mmu->os, Mmu->pageTablePhysical, (gctPOINTER) Mmu->pageTableLogical, Mmu->pageTableSize)); } /* Delete the page table mutex. */ gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex)); #if gcdPROCESS_ADDRESS_SPACE for (i = 0; i < Mmu->mtlbSize / 4; i++) { struct _gcsMMU_STLB *stlb = ((struct _gcsMMU_STLB **)Mmu->stlbs)[i]; if (stlb) { gcmkVERIFY_OK(gckOS_FreeContiguous( Mmu->os, stlb->physical, stlb->logical, stlb->size)); gcmkOS_SAFE_FREE(Mmu->os, stlb); } } gcmkOS_SAFE_FREE(Mmu->os, Mmu->stlbs); #endif /* Mark the gckMMU object as unknown. */ Mmu->object.type = gcvOBJ_UNKNOWN; /* Free the gckMMU object. */ gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu)); /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; } /******************************************************************************* ** _AdjstIndex ** ** Adjust the index from which we search for a usable node to make sure ** index allocated is greater than Start. */ gceSTATUS _AdjustIndex( IN gckMMU Mmu, IN gctUINT32 Index, IN gctUINT32 PageCount, IN gctUINT32 Start, OUT gctUINT32 * IndexAdjusted ) { gceSTATUS status; gctUINT32 index = Index; gctUINT32_PTR map = Mmu->mapLogical; gcmkHEADER(); for (; index < Mmu->pageTableEntries;) { gctUINT32 result = 0; gctUINT32 nodeSize = 0; if (index >= Start) { break; } switch (gcmENTRY_TYPE(map[index])) { case gcvMMU_SINGLE: nodeSize = 1; break; case gcvMMU_FREE: nodeSize = map[index] >> 8; break; default: gcmkFATAL("MMU table correcupted at index %u!", index); gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } if (nodeSize > PageCount) { result = index + (nodeSize - PageCount); if (result >= Start) { break; } } switch (gcmENTRY_TYPE(map[index])) { case gcvMMU_SINGLE: index = map[index] >> 8; break; case gcvMMU_FREE: index = map[index + 1]; break; default: gcmkFATAL("MMU table correcupted at index %u!", index); gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } } *IndexAdjusted = index; gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: gcmkFOOTER(); return status; } gceSTATUS gckMMU_Construct( IN gckKERNEL Kernel, IN gctSIZE_T MmuSize, OUT gckMMU * Mmu ) { #if gcdSHARED_PAGETABLE gceSTATUS status; gctPOINTER pointer; gcmkHEADER_ARG("Kernel=0x%08x", Kernel); if (sharedPageTable == gcvNULL) { gcmkONERROR( gckOS_Allocate(Kernel->os, sizeof(struct _gcsSharedPageTable), &pointer)); sharedPageTable = pointer; gcmkONERROR( gckOS_ZeroMemory(sharedPageTable, sizeof(struct _gcsSharedPageTable))); gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu)); } *Mmu = sharedPageTable->mmu; sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware; sharedPageTable->reference++; gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference); return gcvSTATUS_OK; OnError: if (sharedPageTable) { if (sharedPageTable->mmu) { gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu)); } gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable)); } gcmkFOOTER(); return status; #elif gcdMIRROR_PAGETABLE gceSTATUS status; gctPOINTER pointer; gcmkHEADER_ARG("Kernel=0x%08x", Kernel); if (mirrorPageTable == gcvNULL) { gcmkONERROR( gckOS_Allocate(Kernel->os, sizeof(struct _gcsMirrorPageTable), &pointer)); mirrorPageTable = pointer; gcmkONERROR( gckOS_ZeroMemory(mirrorPageTable, sizeof(struct _gcsMirrorPageTable))); gcmkONERROR( gckOS_CreateMutex(Kernel->os, &mirrorPageTableMutex)); } gcmkONERROR(_Construct(Kernel, MmuSize, Mmu)); mirrorPageTable->mmus[mirrorPageTable->reference] = *Mmu; mirrorPageTable->hardwares[mirrorPageTable->reference] = Kernel->hardware; mirrorPageTable->reference++; gcmkFOOTER_ARG("mirrorPageTable->reference=%lu", mirrorPageTable->reference); return gcvSTATUS_OK; OnError: if (mirrorPageTable && mirrorPageTable->reference == 0) { gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, mirrorPageTable)); } gcmkFOOTER(); return status; #else return _Construct(Kernel, MmuSize, Mmu); #endif } gceSTATUS gckMMU_Destroy( IN gckMMU Mmu ) { #if gcdSHARED_PAGETABLE gckOS os = Mmu->os; sharedPageTable->reference--; if (sharedPageTable->reference == 0) { if (sharedPageTable->mmu) { gcmkVERIFY_OK(_Destroy(Mmu)); } gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, sharedPageTable)); } return gcvSTATUS_OK; #elif gcdMIRROR_PAGETABLE mirrorPageTable->reference--; if (mirrorPageTable->reference == 0) { gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, mirrorPageTable)); gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, mirrorPageTableMutex)); } return _Destroy(Mmu); #else return _Destroy(Mmu); #endif } /******************************************************************************* ** ** gckMMU_AllocatePages ** ** Allocate pages inside the page table. ** ** INPUT: ** ** gckMMU Mmu ** Pointer to an gckMMU object. ** ** gctSIZE_T PageCount ** Number of pages to allocate. ** ** OUTPUT: ** ** gctPOINTER * PageTable ** Pointer to a variable that receives the base address of the page ** table. ** ** gctUINT32 * Address ** Pointer to a variable that receives the hardware specific address. */ gceSTATUS _AllocatePages( IN gckMMU Mmu, IN gctSIZE_T PageCount, IN gceSURF_TYPE Type, OUT gctPOINTER * PageTable, OUT gctUINT32 * Address ) { gceSTATUS status; gctBOOL mutex = gcvFALSE; gctUINT32 index = 0, previous = ~0U, left; gctUINT32_PTR map; gctBOOL gotIt; gctUINT32 address; gctUINT32 pageCount; gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkVERIFY_ARGUMENT(PageCount > 0); gcmkVERIFY_ARGUMENT(PageTable != gcvNULL); if (PageCount > Mmu->pageTableEntries) { /* Not enough pages avaiable. */ gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } gcmkSAFECASTSIZET(pageCount, PageCount); /* Grab the mutex. */ gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); mutex = gcvTRUE; /* Cast pointer to page table. */ for (map = Mmu->mapLogical, gotIt = gcvFALSE; !gotIt;) { index = Mmu->heapList; if ((Mmu->hardware->mmuVersion == 0) && (Type == gcvSURF_VERTEX)) { gcmkONERROR(_AdjustIndex( Mmu, index, pageCount, gcdVERTEX_START / gcmSIZEOF(gctUINT32), &index )); } /* Walk the heap list. */ for (; !gotIt && (index < Mmu->pageTableEntries);) { /* Check the node type. */ switch (gcmENTRY_TYPE(_ReadPageEntry(&map[index]))) { case gcvMMU_SINGLE: /* Single odes are valid if we only need 1 page. */ if (pageCount == 1) { gotIt = gcvTRUE; } else { /* Move to next node. */ previous = index; index = _ReadPageEntry(&map[index]) >> 8; } break; case gcvMMU_FREE: /* Test if the node has enough space. */ if (pageCount <= (_ReadPageEntry(&map[index]) >> 8)) { gotIt = gcvTRUE; } else { /* Move to next node. */ previous = index; index = _ReadPageEntry(&map[index + 1]); } break; default: gcmkFATAL("MMU table correcupted at index %u!", index); gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } } /* Test if we are out of memory. */ if (index >= Mmu->pageTableEntries) { if (Mmu->freeNodes) { /* Time to move out the trash! */ gcmkONERROR(_Collect(Mmu)); /* We are going to search from start, so reset previous to start. */ previous = ~0U; } else { /* Out of resources. */ gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } } } switch (gcmENTRY_TYPE(_ReadPageEntry(&map[index]))) { case gcvMMU_SINGLE: /* Unlink single node from free list. */ gcmkONERROR( _Link(Mmu, previous, _ReadPageEntry(&map[index]) >> 8)); break; case gcvMMU_FREE: /* Check how many pages will be left. */ left = (_ReadPageEntry(&map[index]) >> 8) - pageCount; switch (left) { case 0: /* The entire node is consumed, just unlink it. */ gcmkONERROR( _Link(Mmu, previous, _ReadPageEntry(&map[index + 1]))); break; case 1: /* One page will remain. Convert the node to a single node and ** advance the index. */ _WritePageEntry(&map[index], (_ReadPageEntry(&map[index + 1]) << 8) | gcvMMU_SINGLE); index ++; break; default: /* Enough pages remain for a new node. However, we will just adjust ** the size of the current node and advance the index. */ _WritePageEntry(&map[index], (left << 8) | gcvMMU_FREE); index += left; break; } break; } /* Mark node as used. */ gcmkONERROR(_FillPageTable(&map[index], pageCount, gcvMMU_USED)); /* Return pointer to page table. */ *PageTable = &Mmu->pageTableLogical[index]; /* Build virtual address. */ if (Mmu->hardware->mmuVersion == 0) { gcmkONERROR( gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address)); } else { gctUINT32 masterOffset = index / gcdMMU_STLB_4K_ENTRY_NUM + Mmu->dynamicMappingStart; gctUINT32 slaveOffset = index % gcdMMU_STLB_4K_ENTRY_NUM; address = (masterOffset << gcdMMU_MTLB_SHIFT) | (slaveOffset << gcdMMU_STLB_4K_SHIFT); } if (Address != gcvNULL) { *Address = address; } /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); /* Success. */ gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x", *PageTable, gcmOPT_VALUE(Address)); return gcvSTATUS_OK; OnError: if (mutex) { /* Release the mutex. */ gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); } /* Return the status. */ gcmkFOOTER(); return status; } /******************************************************************************* ** ** gckMMU_FreePages ** ** Free pages inside the page table. ** ** INPUT: ** ** gckMMU Mmu ** Pointer to an gckMMU object. ** ** gctPOINTER PageTable ** Base address of the page table to free. ** ** gctSIZE_T PageCount ** Number of pages to free. ** ** OUTPUT: ** ** Nothing. */ gceSTATUS _FreePages( IN gckMMU Mmu, IN gctPOINTER PageTable, IN gctSIZE_T PageCount ) { gctUINT32_PTR node; gceSTATUS status; gctBOOL acquired = gcvFALSE; gctUINT32 pageCount; gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu", Mmu, PageTable, PageCount); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkVERIFY_ARGUMENT(PageTable != gcvNULL); gcmkVERIFY_ARGUMENT(PageCount > 0); gcmkSAFECASTSIZET(pageCount, PageCount); /* Get the node by index. */ node = Mmu->mapLogical + ((gctUINT32_PTR)PageTable - Mmu->pageTableLogical); gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); acquired = gcvTRUE; #if gcdMMU_CLEAR_VALUE if (Mmu->hardware->mmuVersion == 0) { _FillPageTable(PageTable, pageCount, gcdMMU_CLEAR_VALUE); } #endif if (PageCount == 1) { /* Single page node. */ _WritePageEntry(node, (~((1U<<8)-1)) | gcvMMU_SINGLE); #if gcdUSE_MMU_EXCEPTION /* Enable exception */ _WritePageEntry(PageTable, (1 << 1)); #endif } else { /* Mark the node as free. */ _WritePageEntry(node, (pageCount << 8) | gcvMMU_FREE); _WritePageEntry(node + 1, ~0U); #if gcdUSE_MMU_EXCEPTION /* Enable exception */ gcmkVERIFY_OK(_FillPageTable(PageTable, pageCount, 1 << 1)); #endif } /* We have free nodes. */ Mmu->freeNodes = gcvTRUE; gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); acquired = gcvFALSE; /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: if (acquired) { gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); } gcmkFOOTER(); return status; } gceSTATUS gckMMU_AllocatePages( IN gckMMU Mmu, IN gctSIZE_T PageCount, OUT gctPOINTER * PageTable, OUT gctUINT32 * Address ) { return gckMMU_AllocatePagesEx( Mmu, PageCount, gcvSURF_TYPE_UNKNOWN, PageTable, Address); } gceSTATUS gckMMU_AllocatePagesEx( IN gckMMU Mmu, IN gctSIZE_T PageCount, IN gceSURF_TYPE Type, OUT gctPOINTER * PageTable, OUT gctUINT32 * Address ) { #if gcdMIRROR_PAGETABLE gceSTATUS status; gctPOINTER pageTable; gctUINT32 address; gctINT i; gckMMU mmu; gctBOOL acquired = gcvFALSE; gctBOOL allocated = gcvFALSE; gckOS_AcquireMutex(Mmu->os, mirrorPageTableMutex, gcvINFINITE); acquired = gcvTRUE; /* Allocate page table for current MMU. */ for (i = 0; i < (gctINT)mirrorPageTable->reference; i++) { if (Mmu == mirrorPageTable->mmus[i]) { gcmkONERROR(_AllocatePages(Mmu, PageCount, Type, PageTable, Address)); allocated = gcvTRUE; } } /* Allocate page table for other MMUs. */ for (i = 0; i < (gctINT)mirrorPageTable->reference; i++) { mmu = mirrorPageTable->mmus[i]; if (Mmu != mmu) { gcmkONERROR(_AllocatePages(mmu, PageCount, Type, &pageTable, &address)); gcmkASSERT(address == *Address); } } gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex); acquired = gcvFALSE; return gcvSTATUS_OK; OnError: if (allocated) { /* Page tables for multiple GPU always keep the same. So it is impossible * the fist one allocates successfully but others fail. */ gcmkASSERT(0); } if (acquired) { gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex); } return status; #else return _AllocatePages(Mmu, PageCount, Type, PageTable, Address); #endif } gceSTATUS gckMMU_FreePages( IN gckMMU Mmu, IN gctPOINTER PageTable, IN gctSIZE_T PageCount ) { #if gcdMIRROR_PAGETABLE gctINT i; gctUINT32 offset; gckMMU mmu; gckOS_AcquireMutex(Mmu->os, mirrorPageTableMutex, gcvINFINITE); gcmkVERIFY_OK(_FreePages(Mmu, PageTable, PageCount)); offset = (gctUINT32)PageTable - (gctUINT32)Mmu->pageTableLogical; for (i = 0; i < (gctINT)mirrorPageTable->reference; i++) { mmu = mirrorPageTable->mmus[i]; if (mmu != Mmu) { gcmkVERIFY_OK(_FreePages(mmu, mmu->pageTableLogical + offset/4, PageCount)); } } gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex); return gcvSTATUS_OK; #else return _FreePages(Mmu, PageTable, PageCount); #endif } gceSTATUS gckMMU_SetPage( IN gckMMU Mmu, IN gctPHYS_ADDR_T PageAddress, IN gctUINT32 *PageEntry ) { #if gcdMIRROR_PAGETABLE gctUINT32_PTR pageEntry; gctINT i; gckMMU mmu; gctUINT32 offset = (gctUINT32)PageEntry - (gctUINT32)Mmu->pageTableLogical; #endif gctUINT32 addressExt; gctUINT32 address; gcmkHEADER_ARG("Mmu=0x%x", Mmu); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL); gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF)); /* [31:0]. */ address = (gctUINT32)(PageAddress & 0xFFFFFFFF); /* [39:32]. */ addressExt = (gctUINT32)((PageAddress >> 32) & 0xFF); if (Mmu->hardware->mmuVersion == 0) { _WritePageEntry(PageEntry, address); } else { _WritePageEntry(PageEntry, _SetPage(address, addressExt)); } #if gcdMIRROR_PAGETABLE for (i = 0; i < (gctINT)mirrorPageTable->reference; i++) { mmu = mirrorPageTable->mmus[i]; if (mmu != Mmu) { pageEntry = mmu->pageTableLogical + offset / 4; if (mmu->hardware->mmuVersion == 0) { _WritePageEntry(pageEntry, address); } else { _WritePageEntry(pageEntry, _SetPage(address, addressExt)); } } } #endif /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; } #if gcdPROCESS_ADDRESS_SPACE gceSTATUS gckMMU_GetPageEntry( IN gckMMU Mmu, IN gctUINT32 Address, IN gctUINT32_PTR *PageTable ) { gceSTATUS status; struct _gcsMMU_STLB *stlb; struct _gcsMMU_STLB **stlbs = Mmu->stlbs; gctUINT32 offset = _MtlbOffset(Address); gctUINT32 mtlbEntry; gctBOOL ace = gckHARDWARE_IsFeatureAvailable(Mmu->hardware, gcvFEATURE_ACE); gcmkHEADER_ARG("Mmu=0x%x", Mmu); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkVERIFY_ARGUMENT((Address & 0xFFF) == 0); stlb = stlbs[offset]; if (stlb == gcvNULL) { gcmkONERROR(_AllocateStlb(Mmu->os, &stlb)); mtlbEntry = stlb->physBase | gcdMMU_MTLB_4K_PAGE | gcdMMU_MTLB_PRESENT ; if (ace) { mtlbEntry = mtlbEntry /* Secure */ | (1 << 4); } /* Insert Slave TLB address to Master TLB entry.*/ _WritePageEntry(Mmu->mtlbLogical + offset, mtlbEntry); /* Record stlb. */ stlbs[offset] = stlb; } *PageTable = &stlb->logical[_StlbOffset(Address)]; /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: gcmkFOOTER(); return status; } gceSTATUS _CheckMap( IN gckMMU Mmu ) { gceSTATUS status; gctUINT32_PTR map = Mmu->mapLogical; gctUINT32 index; for (index = Mmu->heapList; index < Mmu->pageTableEntries;) { /* Check the node type. */ switch (gcmENTRY_TYPE(_ReadPageEntry(&map[index]))) { case gcvMMU_SINGLE: /* Move to next node. */ index = _ReadPageEntry(&map[index]) >> 8; break; case gcvMMU_FREE: /* Move to next node. */ index = _ReadPageEntry(&map[index + 1]); break; default: gcmkFATAL("MMU table correcupted at index [%u] = %x!", index, map[index]); gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } } return gcvSTATUS_OK; OnError: return status; } gceSTATUS gckMMU_FlatMapping( IN gckMMU Mmu, IN gctUINT32 Physical ) { gceSTATUS status; gctUINT32 index = _AddressToIndex(Mmu, Physical); gctUINT32 i; gctBOOL gotIt = gcvFALSE; gctUINT32_PTR map = Mmu->mapLogical; gctUINT32 previous = ~0U; gctUINT32_PTR pageTable; gckMMU_GetPageEntry(Mmu, Physical, &pageTable); _WritePageEntry(pageTable, _SetPage(Physical)); if (map) { /* Find node which contains index. */ for (i = 0; !gotIt && (i < Mmu->pageTableEntries);) { gctUINT32 numPages; switch (gcmENTRY_TYPE(_ReadPageEntry(&map[i]))) { case gcvMMU_SINGLE: if (i == index) { gotIt = gcvTRUE; } else { previous = i; i = _ReadPageEntry(&map[i]) >> 8; } break; case gcvMMU_FREE: numPages = _ReadPageEntry(&map[i]) >> 8; if (index >= i && index < i + numPages) { gotIt = gcvTRUE; } else { previous = i; i = _ReadPageEntry(&map[i + 1]); } break; default: gcmkFATAL("MMU table correcupted at index %u!", index); gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES); } } switch (gcmENTRY_TYPE(_ReadPageEntry(&map[i]))) { case gcvMMU_SINGLE: /* Unlink single node from free list. */ gcmkONERROR( _Link(Mmu, previous, _ReadPageEntry(&map[i]) >> 8)); break; case gcvMMU_FREE: /* Split the node. */ { gctUINT32 start; gctUINT32 next = _ReadPageEntry(&map[i+1]); gctUINT32 total = _ReadPageEntry(&map[i]) >> 8; gctUINT32 countLeft = index - i; gctUINT32 countRight = total - countLeft - 1; if (countLeft) { start = i; _AddFree(Mmu, previous, start, countLeft); previous = start; } if (countRight) { start = index + 1; _AddFree(Mmu, previous, start, countRight); previous = start; } _Link(Mmu, previous, next); } break; } } return gcvSTATUS_OK; OnError: /* Roll back. */ return status; } gceSTATUS gckMMU_FreePagesEx( IN gckMMU Mmu, IN gctUINT32 Address, IN gctSIZE_T PageCount ) { gctUINT32_PTR node; gceSTATUS status; #if gcdUSE_MMU_EXCEPTION gctUINT32 i; struct _gcsMMU_STLB *stlb; struct _gcsMMU_STLB **stlbs = Mmu->stlbs; #endif gcmkHEADER_ARG("Mmu=0x%x Address=0x%x PageCount=%lu", Mmu, Address, PageCount); /* Verify the arguments. */ gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkVERIFY_ARGUMENT(PageCount > 0); /* Get the node by index. */ node = Mmu->mapLogical + _AddressToIndex(Mmu, Address); gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE)); if (PageCount == 1) { /* Single page node. */ _WritePageEntry(node, (~((1U<<8)-1)) | gcvMMU_SINGLE); } else { /* Mark the node as free. */ _WritePageEntry(node, (PageCount << 8) | gcvMMU_FREE); _WritePageEntry(node + 1, ~0U); } /* We have free nodes. */ Mmu->freeNodes = gcvTRUE; #if gcdUSE_MMU_EXCEPTION for (i = 0; i < PageCount; i++) { /* Get */ stlb = stlbs[_MtlbOffset(Address)]; /* Enable exception */ stlb->logical[_StlbOffset(Address)] = gcdMMU_STLB_EXCEPTION; } #endif gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex)); /* Success. */ gcmkFOOTER_NO(); return gcvSTATUS_OK; OnError: gcmkFOOTER(); return status; } #endif gceSTATUS gckMMU_Flush( IN gckMMU Mmu, IN gceSURF_TYPE Type ) { gckHARDWARE hardware; gctUINT32 mask; gctINT i; if (Type == gcvSURF_VERTEX || Type == gcvSURF_INDEX) { mask = gcvPAGE_TABLE_DIRTY_BIT_FE; } else { mask = gcvPAGE_TABLE_DIRTY_BIT_OTHER; } #if gcdPROCESS_ADDRESS_SPACE for (i = 0; i < gcdMAX_GPU_COUNT; i++) { gcmkVERIFY_OK( gckOS_AtomSetMask(Mmu->pageTableDirty[i], mask)); } #else #if gcdSHARED_PAGETABLE for (i = 0; i < gcdMAX_GPU_COUNT; i++) { hardware = sharedPageTable->hardwares[i]; if (hardware) { gcmkVERIFY_OK(gckOS_AtomSetMask(hardware->pageTableDirty, mask)); } } #elif gcdMIRROR_PAGETABLE for (i = 0; i < (gctINT)mirrorPageTable->reference; i++) { hardware = mirrorPageTable->hardwares[i]; /* Notify cores who use this page table. */ gcmkVERIFY_OK( gckOS_AtomSetMask(hardware->pageTableDirty, mask)); } #else hardware = Mmu->hardware; gcmkVERIFY_OK( gckOS_AtomSetMask(hardware->pageTableDirty, mask)); #endif #endif return gcvSTATUS_OK; } gceSTATUS gckMMU_DumpPageTableEntry( IN gckMMU Mmu, IN gctUINT32 Address ) { #if gcdPROCESS_ADDRESS_SPACE gcsMMU_STLB_PTR *stlbs = Mmu->stlbs; gcsMMU_STLB_PTR stlbDesc = stlbs[_MtlbOffset(Address)]; #else gctUINT32_PTR pageTable; gctUINT32 index; gctUINT32 mtlb, stlb; #endif gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address); gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU); gcmkASSERT(Mmu->hardware->mmuVersion > 0); #if gcdPROCESS_ADDRESS_SPACE if (stlbDesc) { gcmkPRINT(" STLB entry = 0x%08X", _ReadPageEntry(&stlbDesc->logical[_StlbOffset(Address)])); } else { gcmkPRINT(" MTLB entry is empty."); } #else mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT; if (mtlb >= Mmu->dynamicMappingStart) { stlb = (Address & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT; pageTable = Mmu->pageTableLogical; index = (mtlb - Mmu->dynamicMappingStart) * gcdMMU_STLB_4K_ENTRY_NUM + stlb; gcmkPRINT(" Page table entry = 0x%08X", _ReadPageEntry(pageTable + index)); } else { gcsMMU_STLB_PTR stlbObj = Mmu->staticSTLB; gctUINT32 entry = Mmu->mtlbLogical[mtlb]; stlb = (Address & gcdMMU_STLB_64K_MASK) >> gcdMMU_STLB_64K_SHIFT; entry &= 0xFFFFFFF0; while (stlbObj) { if (entry == stlbObj->physBase) { gcmkPRINT(" Page table entry = 0x%08X", stlbObj->logical[stlb]); break; } stlbObj = stlbObj->next; } } #endif gcmkFOOTER_NO(); return gcvSTATUS_OK; } /****************************************************************************** ****************************** T E S T C O D E ****************************** ******************************************************************************/