mali_kbase_mem.h

Go to the documentation of this file.

/*
 *
 * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */





#ifndef _KBASE_MEM_H_
#define _KBASE_MEM_H_

#ifndef _KBASE_H_
#error "Don't include this file directly, use mali_kbase.h instead"
#endif

#include <linux/kref.h>
#ifdef CONFIG_KDS
#include <linux/kds.h>
#endif              /* CONFIG_KDS */
#ifdef CONFIG_UMP
#include <linux/ump.h>
#endif              /* CONFIG_UMP */
#include "mali_base_kernel.h"
#include <mali_kbase_hw.h>
#include "mali_kbase_pm.h"
#include "mali_kbase_defs.h"
#if defined(CONFIG_MALI_GATOR_SUPPORT)
#include "mali_kbase_gator.h"
#endif
/* Required for kbase_mem_evictable_unmake */
#include "mali_kbase_mem_linux.h"

/* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */

/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages.
The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and
page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table
updates and generates duplicate page faults as the page table information used by the MMU is not valid.   */
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */

#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0)   /* round to 1 page */

/* This must always be a power of 2 */
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2)
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316)
#define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630)

struct kbase_cpu_mapping {
    struct   list_head mappings_list;
    struct   kbase_mem_phy_alloc *alloc;
    struct   kbase_context *kctx;
    struct   kbase_va_region *region;
    int      count;
    int      free_on_close;
};

enum kbase_memory_type {
    KBASE_MEM_TYPE_NATIVE,
    KBASE_MEM_TYPE_IMPORTED_UMP,
    KBASE_MEM_TYPE_IMPORTED_UMM,
    KBASE_MEM_TYPE_IMPORTED_USER_BUF,
    KBASE_MEM_TYPE_ALIAS,
    KBASE_MEM_TYPE_TB,
    KBASE_MEM_TYPE_RAW
};

/* internal structure, mirroring base_mem_aliasing_info,
 * but with alloc instead of a gpu va (handle) */
struct kbase_aliased {
    struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */
    u64 offset; /* in pages */
    u64 length; /* in pages */
};

#define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED  (1ul << 0)
#define KBASE_MEM_PHY_ALLOC_LARGE            (1ul << 1)

/* physical pages tracking object.
 * Set up to track N pages.
 * N not stored here, the creator holds that info.
 * This object only tracks how many elements are actually valid (present).
 * Changing of nents or *pages should only happen if the kbase_mem_phy_alloc is not
 * shared with another region or client. CPU mappings are OK to exist when changing, as
 * long as the tracked mappings objects are updated as part of the change.
 */
struct kbase_mem_phy_alloc {
    struct kref           kref; /* number of users of this alloc */
    atomic_t              gpu_mappings;
    size_t                nents; /* 0..N */
    phys_addr_t           *pages; /* N elements, only 0..nents are valid */

    /* kbase_cpu_mappings */
    struct list_head      mappings;

    /* Node used to store this allocation on the eviction list */
    struct list_head      evict_node;
    /* Physical backing size when the pages where evicted */
    size_t                evicted;
    /*
     * Back reference to the region structure which created this
     * allocation, or NULL if it has been freed.
     */
    struct kbase_va_region *reg;

    /* type of buffer */
    enum kbase_memory_type type;

    unsigned long properties;

    struct list_head       zone_cache;

    /* member in union valid based on @a type */
    union {
#ifdef CONFIG_UMP
        ump_dd_handle ump_handle;
#endif /* CONFIG_UMP */
#if defined(CONFIG_DMA_SHARED_BUFFER)
        struct {
            struct dma_buf *dma_buf;
            struct dma_buf_attachment *dma_attachment;
            unsigned int current_mapping_usage_count;
            struct sg_table *sgt;
        } umm;
#endif /* defined(CONFIG_DMA_SHARED_BUFFER) */
        struct {
            u64 stride;
            size_t nents;
            struct kbase_aliased *aliased;
        } alias;
        /* Used by type = (KBASE_MEM_TYPE_NATIVE, KBASE_MEM_TYPE_TB) */
        struct kbase_context *kctx;
        struct {
            unsigned long address;
            unsigned long size;
            unsigned long nr_pages;
            struct page **pages;
            unsigned int current_mapping_usage_count;
            struct mm_struct *mm;
            dma_addr_t *dma_addrs;
        } user_buf;
    } imported;
};

static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc)
{
    KBASE_DEBUG_ASSERT(alloc);
    /* we only track mappings of NATIVE buffers */
    if (alloc->type == KBASE_MEM_TYPE_NATIVE)
        atomic_inc(&alloc->gpu_mappings);
}

static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc *alloc)
{
    KBASE_DEBUG_ASSERT(alloc);
    /* we only track mappings of NATIVE buffers */
    if (alloc->type == KBASE_MEM_TYPE_NATIVE)
        if (0 > atomic_dec_return(&alloc->gpu_mappings)) {
            pr_err("Mismatched %s:\n", __func__);
            dump_stack();
        }
}

void kbase_mem_kref_free(struct kref *kref);

int kbase_mem_init(struct kbase_device *kbdev);
void kbase_mem_halt(struct kbase_device *kbdev);
void kbase_mem_term(struct kbase_device *kbdev);

static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_get(struct kbase_mem_phy_alloc *alloc)
{
    kref_get(&alloc->kref);
    return alloc;
}

static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_mem_phy_alloc *alloc)
{
    kref_put(&alloc->kref, kbase_mem_kref_free);
    return NULL;
}

struct kbase_va_region {
    struct rb_node rblink;
    struct list_head link;

    struct kbase_context *kctx; /* Backlink to base context */

    u64 start_pfn;      /* The PFN in GPU space */
    size_t nr_pages;

/* Free region */
#define KBASE_REG_FREE              (1ul << 0)
/* CPU write access */
#define KBASE_REG_CPU_WR            (1ul << 1)
/* GPU write access */
#define KBASE_REG_GPU_WR            (1ul << 2)
/* No eXecute flag */
#define KBASE_REG_GPU_NX            (1ul << 3)
/* Is CPU cached? */
#define KBASE_REG_CPU_CACHED        (1ul << 4)
/* Is GPU cached? */
#define KBASE_REG_GPU_CACHED        (1ul << 5)

#define KBASE_REG_GROWABLE          (1ul << 6)
/* Can grow on pf? */
#define KBASE_REG_PF_GROW           (1ul << 7)

/* VA managed by us */
#define KBASE_REG_CUSTOM_VA         (1ul << 8)

/* inner shareable coherency */
#define KBASE_REG_SHARE_IN          (1ul << 9)
/* inner & outer shareable coherency */
#define KBASE_REG_SHARE_BOTH        (1ul << 10)

/* Space for 4 different zones */
#define KBASE_REG_ZONE_MASK         (3ul << 11)
#define KBASE_REG_ZONE(x)           (((x) & 3) << 11)

/* GPU read access */
#define KBASE_REG_GPU_RD            (1ul<<13)
/* CPU read access */
#define KBASE_REG_CPU_RD            (1ul<<14)

/* Index of chosen MEMATTR for this region (0..7) */
#define KBASE_REG_MEMATTR_MASK      (7ul << 16)
#define KBASE_REG_MEMATTR_INDEX(x)  (((x) & 7) << 16)
#define KBASE_REG_MEMATTR_VALUE(x)  (((x) & KBASE_REG_MEMATTR_MASK) >> 16)

#define KBASE_REG_SECURE            (1ul << 19)

#define KBASE_REG_DONT_NEED         (1ul << 20)

#define KBASE_REG_ZONE_SAME_VA      KBASE_REG_ZONE(0)

/* only used with 32-bit clients */
/*
 * On a 32bit platform, custom VA should be wired from (4GB + shader region)
 * to the VA limit of the GPU. Unfortunately, the Linux mmap() interface
 * limits us to 2^32 pages (2^44 bytes, see mmap64 man page for reference).
 * So we put the default limit to the maximum possible on Linux and shrink
 * it down, if required by the GPU, during initialization.
 */

/*
 * Dedicated 16MB region for shader code:
 * VA range 0x101000000-0x102000000
 */
#define KBASE_REG_ZONE_EXEC         KBASE_REG_ZONE(1)
#define KBASE_REG_ZONE_EXEC_BASE    (0x101000000ULL >> PAGE_SHIFT)
#define KBASE_REG_ZONE_EXEC_SIZE    ((16ULL * 1024 * 1024) >> PAGE_SHIFT)

#define KBASE_REG_ZONE_CUSTOM_VA         KBASE_REG_ZONE(2)
#define KBASE_REG_ZONE_CUSTOM_VA_BASE    (KBASE_REG_ZONE_EXEC_BASE + KBASE_REG_ZONE_EXEC_SIZE) /* Starting after KBASE_REG_ZONE_EXEC */
#define KBASE_REG_ZONE_CUSTOM_VA_SIZE    (((1ULL << 44) >> PAGE_SHIFT) - KBASE_REG_ZONE_CUSTOM_VA_BASE)
/* end 32-bit clients only */

    unsigned long flags;

    size_t extent; /* nr of pages alloc'd on PF */

    struct kbase_mem_phy_alloc *cpu_alloc; /* the one alloc object we mmap to the CPU when mapping this region */
    struct kbase_mem_phy_alloc *gpu_alloc; /* the one alloc object we mmap to the GPU when mapping this region */

    /* non-NULL if this memory object is a kds_resource */
    struct kds_resource *kds_res;

    /* List head used to store the region in the JIT allocation pool */
    struct list_head jit_node;
};

/* Common functions */
static inline phys_addr_t *kbase_get_cpu_phy_pages(struct kbase_va_region *reg)
{
    KBASE_DEBUG_ASSERT(reg);
    KBASE_DEBUG_ASSERT(reg->cpu_alloc);
    KBASE_DEBUG_ASSERT(reg->gpu_alloc);
    KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);

    return reg->cpu_alloc->pages;
}

static inline phys_addr_t *kbase_get_gpu_phy_pages(struct kbase_va_region *reg)
{
    KBASE_DEBUG_ASSERT(reg);
    KBASE_DEBUG_ASSERT(reg->cpu_alloc);
    KBASE_DEBUG_ASSERT(reg->gpu_alloc);
    KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);

    return reg->gpu_alloc->pages;
}

static inline size_t kbase_reg_current_backed_size(struct kbase_va_region *reg)
{
    KBASE_DEBUG_ASSERT(reg);
    /* if no alloc object the backed size naturally is 0 */
    if (!reg->cpu_alloc)
        return 0;

    KBASE_DEBUG_ASSERT(reg->cpu_alloc);
    KBASE_DEBUG_ASSERT(reg->gpu_alloc);
    KBASE_DEBUG_ASSERT(reg->cpu_alloc->nents == reg->gpu_alloc->nents);

    return reg->cpu_alloc->nents;
}

#define KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD ((size_t)(4*1024)) /* size above which vmalloc is used over kmalloc */

static inline struct kbase_mem_phy_alloc *kbase_alloc_create(size_t nr_pages, enum kbase_memory_type type)
{
    struct kbase_mem_phy_alloc *alloc;
    size_t alloc_size = sizeof(*alloc) + sizeof(*alloc->pages) * nr_pages;
    size_t per_page_size = sizeof(*alloc->pages);

    /* Imported pages may have page private data already in use */
    if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) {
        alloc_size += nr_pages *
                sizeof(*alloc->imported.user_buf.dma_addrs);
        per_page_size += sizeof(*alloc->imported.user_buf.dma_addrs);
    }

    /*
     * Prevent nr_pages*per_page_size + sizeof(*alloc) from
     * wrapping around.
     */
    if (nr_pages > ((((size_t) -1) - sizeof(*alloc))
            / per_page_size))
        return ERR_PTR(-ENOMEM);

    /* Allocate based on the size to reduce internal fragmentation of vmem */
    if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD)
        alloc = vzalloc(alloc_size);
    else
        alloc = kzalloc(alloc_size, GFP_KERNEL);

    if (!alloc)
        return ERR_PTR(-ENOMEM);

    /* Store allocation method */
    if (alloc_size > KBASE_MEM_PHY_ALLOC_LARGE_THRESHOLD)
        alloc->properties |= KBASE_MEM_PHY_ALLOC_LARGE;

    kref_init(&alloc->kref);
    atomic_set(&alloc->gpu_mappings, 0);
    alloc->nents = 0;
    alloc->pages = (void *)(alloc + 1);
    INIT_LIST_HEAD(&alloc->mappings);
    alloc->type = type;
    INIT_LIST_HEAD(&alloc->zone_cache);

    if (type == KBASE_MEM_TYPE_IMPORTED_USER_BUF)
        alloc->imported.user_buf.dma_addrs =
                (void *) (alloc->pages + nr_pages);

    return alloc;
}

static inline int kbase_reg_prepare_native(struct kbase_va_region *reg,
        struct kbase_context *kctx)
{
    KBASE_DEBUG_ASSERT(reg);
    KBASE_DEBUG_ASSERT(!reg->cpu_alloc);
    KBASE_DEBUG_ASSERT(!reg->gpu_alloc);
    KBASE_DEBUG_ASSERT(reg->flags & KBASE_REG_FREE);

    reg->cpu_alloc = kbase_alloc_create(reg->nr_pages,
            KBASE_MEM_TYPE_NATIVE);
    if (IS_ERR(reg->cpu_alloc))
        return PTR_ERR(reg->cpu_alloc);
    else if (!reg->cpu_alloc)
        return -ENOMEM;
    reg->cpu_alloc->imported.kctx = kctx;
    INIT_LIST_HEAD(&reg->cpu_alloc->evict_node);
    if (kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE)
        && (reg->flags & KBASE_REG_CPU_CACHED)) {
        reg->gpu_alloc = kbase_alloc_create(reg->nr_pages,
                KBASE_MEM_TYPE_NATIVE);
        reg->gpu_alloc->imported.kctx = kctx;
        INIT_LIST_HEAD(&reg->gpu_alloc->evict_node);
    } else {
        reg->gpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc);
    }

    INIT_LIST_HEAD(&reg->jit_node);
    reg->flags &= ~KBASE_REG_FREE;
    return 0;
}

static inline int kbase_atomic_add_pages(int num_pages, atomic_t *used_pages)
{
    int new_val = atomic_add_return(num_pages, used_pages);
#if defined(CONFIG_MALI_GATOR_SUPPORT)
    kbase_trace_mali_total_alloc_pages_change((long long int)new_val);
#endif
    return new_val;
}

static inline int kbase_atomic_sub_pages(int num_pages, atomic_t *used_pages)
{
    int new_val = atomic_sub_return(num_pages, used_pages);
#if defined(CONFIG_MALI_GATOR_SUPPORT)
    kbase_trace_mali_total_alloc_pages_change((long long int)new_val);
#endif
    return new_val;
}

/*
 * Max size for kbdev memory pool (in pages)
 */
#define KBASE_MEM_POOL_MAX_SIZE_KBDEV (SZ_64M >> PAGE_SHIFT)

/*
 * Max size for kctx memory pool (in pages)
 */
//#define KBASE_MEM_POOL_MAX_SIZE_KCTX  (SZ_64M >> PAGE_SHIFT)
#define KBASE_MEM_POOL_MAX_SIZE_KCTX  (SZ_4M >> PAGE_SHIFT)

int kbase_mem_pool_init(struct kbase_mem_pool *pool,
        size_t max_size,
        struct kbase_device *kbdev,
        struct kbase_mem_pool *next_pool);

void kbase_mem_pool_term(struct kbase_mem_pool *pool);

struct page *kbase_mem_pool_alloc(struct kbase_mem_pool *pool);

void kbase_mem_pool_free(struct kbase_mem_pool *pool, struct page *page,
        bool dirty);

int kbase_mem_pool_alloc_pages(struct kbase_mem_pool *pool, size_t nr_pages,
        phys_addr_t *pages);

void kbase_mem_pool_free_pages(struct kbase_mem_pool *pool, size_t nr_pages,
        phys_addr_t *pages, bool dirty, bool reclaimed);

static inline size_t kbase_mem_pool_size(struct kbase_mem_pool *pool)
{
    return ACCESS_ONCE(pool->cur_size);
}

static inline size_t kbase_mem_pool_max_size(struct kbase_mem_pool *pool)
{
    return pool->max_size;
}


void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size);

int kbase_mem_pool_grow(struct kbase_mem_pool *pool, size_t nr_to_grow);

void kbase_mem_pool_trim(struct kbase_mem_pool *pool, size_t new_size);

/*
 * kbase_mem_alloc_page - Allocate a new page for a device
 * @kbdev: The kbase device
 *
 * Most uses should use kbase_mem_pool_alloc to allocate a page. However that
 * function can fail in the event the pool is empty.
 *
 * Return: A new page or NULL if no memory
 */
struct page *kbase_mem_alloc_page(struct kbase_device *kbdev);

int kbase_region_tracker_init(struct kbase_context *kctx);
int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages);
void kbase_region_tracker_term(struct kbase_context *kctx);

struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address(struct kbase_context *kctx, u64 gpu_addr);

struct kbase_va_region *kbase_region_tracker_find_region_base_address(struct kbase_context *kctx, u64 gpu_addr);

struct kbase_va_region *kbase_alloc_free_region(struct kbase_context *kctx, u64 start_pfn, size_t nr_pages, int zone);
void kbase_free_alloced_region(struct kbase_va_region *reg);
int kbase_add_va_region(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);

bool kbase_check_alloc_flags(unsigned long flags);
bool kbase_check_import_flags(unsigned long flags);

int kbase_update_region_flags(struct kbase_context *kctx,
        struct kbase_va_region *reg, unsigned long flags);

void kbase_gpu_vm_lock(struct kbase_context *kctx);
void kbase_gpu_vm_unlock(struct kbase_context *kctx);

int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size);

int kbase_mmu_init(struct kbase_context *kctx);
void kbase_mmu_term(struct kbase_context *kctx);

phys_addr_t kbase_mmu_alloc_pgd(struct kbase_context *kctx);
void kbase_mmu_free_pgd(struct kbase_context *kctx);
int kbase_mmu_insert_pages_no_flush(struct kbase_context *kctx, u64 vpfn,
                  phys_addr_t *phys, size_t nr,
                  unsigned long flags);
int kbase_mmu_insert_pages(struct kbase_context *kctx, u64 vpfn,
                  phys_addr_t *phys, size_t nr,
                  unsigned long flags);
int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn,
                    phys_addr_t phys, size_t nr,
                    unsigned long flags);

int kbase_mmu_teardown_pages(struct kbase_context *kctx, u64 vpfn, size_t nr);
int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, phys_addr_t *phys, size_t nr, unsigned long flags);

int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align);

int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg);

void kbase_mmu_update(struct kbase_context *kctx);

void kbase_mmu_disable(struct kbase_context *kctx);

void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr);

void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);

void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages);

int kbase_sync_now(struct kbase_context *kctx, struct base_syncset *syncset);
void kbase_sync_single(struct kbase_context *kctx, phys_addr_t cpu_pa,
        phys_addr_t gpu_pa, off_t offset, size_t size,
        enum kbase_sync_type sync_fn);
void kbase_pre_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr);
void kbase_post_job_sync(struct kbase_context *kctx, struct base_syncset *syncsets, size_t nr);

/* OS specific functions */
int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr);
int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *reg);
void kbase_os_mem_map_lock(struct kbase_context *kctx);
void kbase_os_mem_map_unlock(struct kbase_context *kctx);

void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages);

static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages)
{
    kbasep_os_process_page_usage_update(kctx, pages);
}

static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages)
{
    kbasep_os_process_page_usage_update(kctx, 0 - pages);
}

int kbasep_find_enclosing_cpu_mapping_offset(
        struct kbase_context *kctx,
        unsigned long uaddr, size_t size, u64 *offset);

enum hrtimer_restart kbasep_as_poke_timer_callback(struct hrtimer *timer);
void kbase_as_poking_timer_retain_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom);
void kbase_as_poking_timer_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbase_jd_atom *katom);

int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_requested);

int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free);

static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr)
{
    SetPagePrivate(p);
    if (sizeof(dma_addr_t) > sizeof(p->private)) {
        /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the
         * private field stays the same. So we have to be clever and
         * use the fact that we only store DMA addresses of whole pages,
         * so the low bits should be zero */
        KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1)));
        set_page_private(p, dma_addr >> PAGE_SHIFT);
    } else {
        set_page_private(p, dma_addr);
    }
}

static inline dma_addr_t kbase_dma_addr(struct page *p)
{
    if (sizeof(dma_addr_t) > sizeof(p->private))
        return ((dma_addr_t)page_private(p)) << PAGE_SHIFT;

    return (dma_addr_t)page_private(p);
}

static inline void kbase_clear_dma_addr(struct page *p)
{
    ClearPagePrivate(p);
}

void kbase_mmu_interrupt_process(struct kbase_device *kbdev,
        struct kbase_context *kctx, struct kbase_as *as);

void page_fault_worker(struct work_struct *data);

void bus_fault_worker(struct work_struct *data);

void kbase_flush_mmu_wqs(struct kbase_device *kbdev);

void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle,
        size_t size, enum dma_data_direction dir);

void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle,
        size_t size, enum dma_data_direction dir);

#ifdef CONFIG_HISI_DEBUG_FS

void kbase_jit_debugfs_init(struct kbase_context *kctx);
#endif /* CONFIG_HISI_DEBUG_FS */

int kbase_jit_init(struct kbase_context *kctx);

struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
        struct base_jit_alloc_info *info);

void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg);

void kbase_jit_backing_lost(struct kbase_va_region *reg);

bool kbase_jit_evict(struct kbase_context *kctx);

void kbase_jit_term(struct kbase_context *kctx);

struct kbase_mem_phy_alloc *kbase_map_external_resource(
        struct kbase_context *kctx, struct kbase_va_region *reg,
        struct mm_struct *locked_mm
#ifdef CONFIG_KDS
        , u32 *kds_res_count, struct kds_resource **kds_resources,
        unsigned long *kds_access_bitmap, bool exclusive
#endif
        );

void kbase_unmap_external_resource(struct kbase_context *kctx,
        struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc);

int kbase_sticky_resource_init(struct kbase_context *kctx);

struct kbase_ctx_ext_res_meta *kbase_sticky_resource_acquire(
        struct kbase_context *kctx, u64 gpu_addr);

bool kbase_sticky_resource_release(struct kbase_context *kctx,
        struct kbase_ctx_ext_res_meta *meta, u64 gpu_addr);

void kbase_sticky_resource_term(struct kbase_context *kctx);

int kbase_zone_cache_update(struct kbase_mem_phy_alloc *alloc,
        size_t start_offset);

int kbase_zone_cache_build(struct kbase_mem_phy_alloc *alloc);

void kbase_zone_cache_clear(struct kbase_mem_phy_alloc *alloc);

#endif              /* _KBASE_MEM_H_ */