mali_kbase_defs.h

Go to the documentation of this file.

/*
 *
 * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */





#ifndef _KBASE_DEFS_H_
#define _KBASE_DEFS_H_

#include <mali_kbase_config.h>
#include <mali_base_hwconfig_features.h>
#include <mali_base_hwconfig_issues.h>
#include <mali_kbase_mem_lowlevel.h>
#include <mali_kbase_mmu_hw.h>
#include <mali_kbase_mmu_mode.h>
#include <mali_kbase_instr_defs.h>
#include <mali_kbase_pm.h>

#include <linux/atomic.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/file.h>

#ifdef CONFIG_MALI_FPGA_BUS_LOGGER
#include <linux/bus_logger.h>
#endif


#ifdef CONFIG_KDS
#include <linux/kds.h>
#endif              /* CONFIG_KDS */

#ifdef CONFIG_SYNC
#include "sync.h"
#endif              /* CONFIG_SYNC */

#include "mali_kbase_dma_fence.h"

#ifdef CONFIG_HISI_DEBUG_FS
#include <linux/debugfs.h>
#endif              /* CONFIG_HISI_DEBUG_FS */

#ifdef CONFIG_PM_DEVFREQ
#include <linux/devfreq.h>
#endif /* CONFIG_DEVFREQ */

#include <linux/clk.h>
#include <linux/regulator/consumer.h>

#if defined(CONFIG_PM_RUNTIME) || \
    (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0))
#define KBASE_PM_RUNTIME 1
#endif

#ifdef CONFIG_MALI_MIDGARD_ENABLE_TRACE
#define KBASE_TRACE_ENABLE 1
#endif

#ifndef KBASE_TRACE_ENABLE
#ifdef CONFIG_MALI_DEBUG
#define KBASE_TRACE_ENABLE 1
#else
#define KBASE_TRACE_ENABLE 0
#endif              /* CONFIG_MALI_DEBUG */
#endif              /* KBASE_TRACE_ENABLE */

#define KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR 1

#define ZAP_TIMEOUT             1000

#define RESET_TIMEOUT           500

#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0

#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0

#define BASE_JM_MAX_NR_SLOTS        3

#define BASE_MAX_NR_AS              16

/* mmu */
#define MIDGARD_MMU_VA_BITS 48

#if MIDGARD_MMU_VA_BITS > 39
#define MIDGARD_MMU_TOPLEVEL    0
#else
#define MIDGARD_MMU_TOPLEVEL    1
#endif

#define MIDGARD_MMU_BOTTOMLEVEL 3

#define GROWABLE_FLAGS_REQUIRED (KBASE_REG_PF_GROW | KBASE_REG_GPU_WR)

#define KBASEP_AS_NR_INVALID     (-1)

#define KBASE_LOCK_REGION_MAX_SIZE (63)
#define KBASE_LOCK_REGION_MIN_SIZE (11)

#define KBASE_TRACE_SIZE_LOG2 8 /* 256 entries */
#define KBASE_TRACE_SIZE (1 << KBASE_TRACE_SIZE_LOG2)
#define KBASE_TRACE_MASK ((1 << KBASE_TRACE_SIZE_LOG2)-1)

#include "mali_kbase_js_defs.h"
#include "mali_kbase_hwaccess_defs.h"

#define KBASEP_FORCE_REPLAY_DISABLED 0

/* Maximum force replay limit when randomization is enabled */
#define KBASEP_FORCE_REPLAY_RANDOM_LIMIT 16

#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED (1<<1)

#define KBASE_KATOM_FLAGS_RERUN (1<<2)
#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)

#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)

#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
/* Atom blocked on cross-slot dependency */
#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
/* Atom has fail dependency on cross-slot dependency */
#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
/* Atom is currently in the list of atoms blocked on cross-slot dependencies */
#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9)
/* Atom is currently holding a context reference */
#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10)
/* Atom requires GPU to be in protected mode */
#define KBASE_KATOM_FLAG_PROTECTED (1<<11)
/* Atom has been stored in runnable_tree */
#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12)

/* SW related flags about types of JS_COMMAND action
 * NOTE: These must be masked off by JS_COMMAND_MASK */

#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100

#define JS_COMMAND_SW_BITS  (JS_COMMAND_SW_CAUSES_DISJOINT)

#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
#error JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK. Must update JS_COMMAND_SW_<..> bitmasks
#endif

#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
        (JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)

#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT

/* Serialize atoms within a slot (ie only one atom per job slot) */
#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0)
/* Serialize atoms between slots (ie only one job slot running at any time) */
#define KBASE_SERIALIZE_INTER_SLOT (1 << 1)
/* Reset the GPU after each atom completion */
#define KBASE_SERIALIZE_RESET (1 << 2)

#ifdef CONFIG_HISI_DEBUG_FS
struct base_job_fault_event {

    u32 event_code;
    struct kbase_jd_atom *katom;
    struct work_struct job_fault_work;
    struct list_head head;
    int reg_offset;
};

#endif

struct kbase_jd_atom_dependency {
    struct kbase_jd_atom *atom;
    u8 dep_type;
};

struct kbase_io_access {
    uintptr_t addr;
    u32 value;
};

struct kbase_io_history {
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
    bool enabled;
#else
    u32 enabled;
#endif

    spinlock_t lock;
    size_t count;
    u16 size;
    struct kbase_io_access *buf;
};

static inline const struct kbase_jd_atom * kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
{
    LOCAL_ASSERT(dep != NULL);

    return (const struct kbase_jd_atom *)(dep->atom);
}

static inline u8 kbase_jd_katom_dep_type(const struct kbase_jd_atom_dependency *dep)
{
    LOCAL_ASSERT(dep != NULL);

    return dep->dep_type;
}

static inline void kbase_jd_katom_dep_set(const struct kbase_jd_atom_dependency *const_dep,
        struct kbase_jd_atom *a, u8 type)
{
    struct kbase_jd_atom_dependency *dep;

    LOCAL_ASSERT(const_dep != NULL);

    dep = (struct kbase_jd_atom_dependency *)const_dep;

    dep->atom = a;
    dep->dep_type = type;
}

static inline void kbase_jd_katom_dep_clear(const struct kbase_jd_atom_dependency *const_dep)
{
    struct kbase_jd_atom_dependency *dep;

    LOCAL_ASSERT(const_dep != NULL);

    dep = (struct kbase_jd_atom_dependency *)const_dep;

    dep->atom = NULL;
    dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
}

enum kbase_atom_gpu_rb_state {
    /* Atom is not currently present in slot ringbuffer */
    KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
    /* Atom is in slot ringbuffer but is blocked on a previous atom */
    KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
    /* Atom is in slot ringbuffer but is waiting for a previous protected
     * mode transition to complete */
    KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV,
    /* Atom is in slot ringbuffer but is waiting for proected mode
     * transition */
    KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION,
    /* Atom is in slot ringbuffer but is waiting for cores to become
     * available */
    KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
    /* Atom is in slot ringbuffer but is blocked on affinity */
    KBASE_ATOM_GPU_RB_WAITING_AFFINITY,
    /* Atom is in slot ringbuffer and ready to run */
    KBASE_ATOM_GPU_RB_READY,
    /* Atom is in slot ringbuffer and has been submitted to the GPU */
    KBASE_ATOM_GPU_RB_SUBMITTED,
    /* Atom must be returned to JS as soon as it reaches the head of the
     * ringbuffer due to a previous failure */
    KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1
};

enum kbase_atom_enter_protected_state {
    /*
     * Starting state:
     * Check if a transition into protected mode is required.
     *
     * NOTE: The integer value of this must
     *       match KBASE_ATOM_EXIT_PROTECTED_CHECK.
     */
    KBASE_ATOM_ENTER_PROTECTED_CHECK = 0,
    /* Wait for vinstr to suspend. */
    KBASE_ATOM_ENTER_PROTECTED_VINSTR,
    /* Wait for the L2 to become idle in preparation for
     * the coherency change. */
    KBASE_ATOM_ENTER_PROTECTED_IDLE_L2,
    /* End state;
     * Prepare coherency change. */
    KBASE_ATOM_ENTER_PROTECTED_FINISHED,
};

enum kbase_atom_exit_protected_state {
    /*
     * Starting state:
     * Check if a transition out of protected mode is required.
     *
     * NOTE: The integer value of this must
     *       match KBASE_ATOM_ENTER_PROTECTED_CHECK.
     */
    KBASE_ATOM_EXIT_PROTECTED_CHECK = 0,
    /* Wait for the L2 to become idle in preparation
     * for the reset. */
    KBASE_ATOM_EXIT_PROTECTED_IDLE_L2,
    /* Issue the protected reset. */
    KBASE_ATOM_EXIT_PROTECTED_RESET,
    /* End state;
     * Wait for the reset to complete. */
    KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT,
};

struct kbase_ext_res {
    u64 gpu_address;
    struct kbase_mem_phy_alloc *alloc;
};

struct kbase_jd_atom {
    struct work_struct work;
    ktime_t start_timestamp;

    struct base_jd_udata udata;
    struct kbase_context *kctx;

    struct list_head dep_head[2];
    struct list_head dep_item[2];
    const struct kbase_jd_atom_dependency dep[2];
    /* List head used during job dispatch job_done processing - as
     * dependencies may not be entirely resolved at this point, we need to
     * use a separate list head. */
    struct list_head jd_item;
    /* true if atom's jd_item is currently on a list. Prevents atom being
     * processed twice. */
    bool in_jd_list;

    u16 nr_extres;
    struct kbase_ext_res *extres;

    u32 device_nr;
    u64 affinity;
    u64 jc;
    enum kbase_atom_coreref_state coreref_state;
#ifdef CONFIG_KDS
    struct list_head node;
    struct kds_resource_set *kds_rset;
    bool kds_dep_satisfied;
#endif              /* CONFIG_KDS */
#ifdef CONFIG_SYNC
    struct sync_fence *fence;
    struct sync_fence_waiter sync_waiter;
#endif              /* CONFIG_SYNC */
#ifdef CONFIG_MALI_DMA_FENCE
    struct {
        /* This points to the dma-buf fence for this atom. If this is
         * NULL then there is no fence for this atom and the other
         * fields related to dma_fence may have invalid data.
         *
         * The context and seqno fields contain the details for this
         * fence.
         *
         * This fence is signaled when the katom is completed,
         * regardless of the event_code of the katom (signal also on
         * failure).
         */
        struct fence *fence;
        /* The dma-buf fence context number for this atom. A unique
         * context number is allocated to each katom in the context on
         * context creation.
         */
        unsigned int context;
        /* The dma-buf fence sequence number for this atom. This is
         * increased every time this katom uses dma-buf fence.
         */
        atomic_t seqno;
        /* This contains a list of all callbacks set up to wait on
         * other fences.  This atom must be held back from JS until all
         * these callbacks have been called and dep_count have reached
         * 0. The initial value of dep_count must be equal to the
         * number of callbacks on this list.
         *
         * This list is protected by jctx.lock. Callbacks are added to
         * this list when the atom is built and the wait are set up.
         * All the callbacks then stay on the list until all callbacks
         * have been called and the atom is queued, or cancelled, and
         * then all callbacks are taken off the list and freed.
         */
        struct list_head callbacks;
        /* Atomic counter of number of outstandind dma-buf fence
         * dependencies for this atom. When dep_count reaches 0 the
         * atom may be queued.
         *
         * The special value "-1" may only be set after the count
         * reaches 0, while holding jctx.lock. This indicates that the
         * atom has been handled, either queued in JS or cancelled.
         *
         * If anyone but the dma-fence worker sets this to -1 they must
         * ensure that any potentially queued worker must have
         * completed before allowing the atom to be marked as unused.
         * This can be done by flushing the fence work queue:
         * kctx->dma_fence.wq.
         */
        atomic_t dep_count;
    } dma_fence;
#endif /* CONFIG_MALI_DMA_FENCE */

    /* Note: refer to kbasep_js_atom_retained_state, which will take a copy of some of the following members */
    enum base_jd_event_code event_code;
    base_jd_core_req core_req;      
    int retry_submit_on_slot;

    u32 ticks;
    /* JS atom priority with respect to other atoms on its kctx. */
    int sched_priority;

    int poking;     /* BASE_HW_ISSUE_8316 */

    wait_queue_head_t completed;
    enum kbase_jd_atom_state status;
#ifdef CONFIG_GPU_TRACEPOINTS
    int work_id;
#endif
    /* Assigned after atom is completed. Used to check whether PRLAM-10676 workaround should be applied */
    int slot_nr;

    u32 atom_flags;

    /* Number of times this atom has been retried. Used by replay soft job.
     */
    int retry_count;

    enum kbase_atom_gpu_rb_state gpu_rb_state;

    u64 need_cache_flush_cores_retained;

    atomic_t blocked;

    /* Pointer to atom that this atom has same-slot dependency on */
    struct kbase_jd_atom *pre_dep;
    /* Pointer to atom that has same-slot dependency on this atom */
    struct kbase_jd_atom *post_dep;

    /* Pointer to atom that this atom has cross-slot dependency on */
    struct kbase_jd_atom *x_pre_dep;
    /* Pointer to atom that has cross-slot dependency on this atom */
    struct kbase_jd_atom *x_post_dep;

    /* The GPU's flush count recorded at the time of submission, used for
     * the cache flush optimisation */
    u32 flush_id;

    struct kbase_jd_atom_backend backend;
#ifdef CONFIG_HISI_DEBUG_FS
    struct base_job_fault_event fault_event;
#endif

    /* List head used for three different purposes:
     *  1. Overflow list for JS ring buffers. If an atom is ready to run,
     *     but there is no room in the JS ring buffer, then the atom is put
     *     on the ring buffer's overflow list using this list node.
     *  2. List of waiting soft jobs.
     */
    struct list_head queue;

    /* Used to keep track of all JIT free/alloc jobs in submission order
     */
    struct list_head jit_node;
    bool jit_blocked;

    /* If non-zero, this indicates that the atom will fail with the set
     * event_code when the atom is processed. */
    enum base_jd_event_code will_fail_event_code;

    /* Atoms will only ever be transitioning into, or out of
     * protected mode so we do not need two separate fields.
     */
    union {
        enum kbase_atom_enter_protected_state enter;
        enum kbase_atom_exit_protected_state exit;
    } protected_state;

    struct rb_node runnable_tree_node;

    /* 'Age' of atom relative to other atoms in the context. */
    u32 age;
};

static inline bool kbase_jd_katom_is_protected(const struct kbase_jd_atom *katom)
{
    return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED);
}

/*
 * Theory of operations:
 *
 * Atom objects are statically allocated within the context structure.
 *
 * Each atom is the head of two lists, one for the "left" set of dependencies, one for the "right" set.
 */

#define KBASE_JD_DEP_QUEUE_SIZE 256

struct kbase_jd_context {
    struct mutex lock;
    struct kbasep_js_kctx_info sched_info;
    struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];

    u32 job_nr;

    wait_queue_head_t zero_jobs_wait;

    struct workqueue_struct *job_done_wq;

    spinlock_t tb_lock;
    u32 *tb;
    size_t tb_wrap_offset;

#ifdef CONFIG_KDS
    struct kds_callback kds_cb;
#endif              /* CONFIG_KDS */
#ifdef CONFIG_GPU_TRACEPOINTS
    atomic_t work_id;
#endif
};

struct kbase_device_info {
    u32 features;
};

enum {
    KBASE_AS_POKE_STATE_IN_FLIGHT     = 1<<0,
    KBASE_AS_POKE_STATE_KILLING_POKE  = 1<<1
};

typedef u32 kbase_as_poke_state;

struct kbase_mmu_setup {
    u64 transtab;
    u64 memattr;
    u64 transcfg;
};

struct kbase_as {
    int number;

    struct workqueue_struct *pf_wq;
    struct work_struct work_pagefault;
    struct work_struct work_busfault;
    enum kbase_mmu_fault_type fault_type;
    bool protected_mode;
    u32 fault_status;
    u64 fault_addr;
    u64 fault_extra_addr;

    struct kbase_mmu_setup current_setup;

    /* BASE_HW_ISSUE_8316  */
    struct workqueue_struct *poke_wq;
    struct work_struct poke_work;
    int poke_refcount;
    kbase_as_poke_state poke_state;
    struct hrtimer poke_timer;
};

static inline int kbase_as_has_bus_fault(struct kbase_as *as)
{
    return as->fault_type == KBASE_MMU_FAULT_TYPE_BUS;
}

static inline int kbase_as_has_page_fault(struct kbase_as *as)
{
    return as->fault_type == KBASE_MMU_FAULT_TYPE_PAGE;
}

struct kbasep_mem_device {
    atomic_t used_pages;   /* Tracks usage of OS shared memory. Updated
                   when OS memory is allocated/freed. */

};

#define KBASE_TRACE_CODE(X) KBASE_TRACE_CODE_ ## X

enum kbase_trace_code {
    /* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
     * THIS MUST BE USED AT THE START OF THE ENUM */
#define KBASE_TRACE_CODE_MAKE_CODE(X) KBASE_TRACE_CODE(X)
#include "mali_kbase_trace_defs.h"
#undef  KBASE_TRACE_CODE_MAKE_CODE
    /* Comma on its own, to extend the list */
    ,
    /* Must be the last in the enum */
    KBASE_TRACE_CODE_COUNT
};

#define KBASE_TRACE_FLAG_REFCOUNT (((u8)1) << 0)
#define KBASE_TRACE_FLAG_JOBSLOT  (((u8)1) << 1)

struct kbase_trace {
    struct timespec timestamp;
    u32 thread_id;
    u32 cpu;
    void *ctx;
    bool katom;
    int atom_number;
    u64 atom_udata[2];
    u64 gpu_addr;
    unsigned long info_val;
    u8 code;
    u8 jobslot;
    u8 refcount;
    u8 flags;
};

enum kbase_timeline_pm_event {
    /* helper for tests */
    KBASEP_TIMELINE_PM_EVENT_FIRST,

    KBASE_TIMELINE_PM_EVENT_RESERVED_0 = KBASEP_TIMELINE_PM_EVENT_FIRST,

    KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED,

    KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE,

    KBASE_TIMELINE_PM_EVENT_GPU_IDLE,

    KBASE_TIMELINE_PM_EVENT_RESERVED_4,

    KBASE_TIMELINE_PM_EVENT_RESERVED_5,

    KBASE_TIMELINE_PM_EVENT_RESERVED_6,

    KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE,

    KBASEP_TIMELINE_PM_EVENT_LAST = KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE,
};

#ifdef CONFIG_MALI_TRACE_TIMELINE
struct kbase_trace_kctx_timeline {
    atomic_t jd_atoms_in_flight;
    u32 owner_tgid;
};

struct kbase_trace_kbdev_timeline {
    /* Note: strictly speaking, not needed, because it's in sync with
     * kbase_device::jm_slots[]::submitted_nr
     *
     * But it's kept as an example of how to add global timeline tracking
     * information
     *
     * The caller must hold hwaccess_lock when accessing this */
    u8 slot_atoms_submitted[BASE_JM_MAX_NR_SLOTS];

    /* Last UID for each PM event */
    atomic_t pm_event_uid[KBASEP_TIMELINE_PM_EVENT_LAST+1];
    /* Counter for generating PM event UIDs */
    atomic_t pm_event_uid_counter;
    /*
     * L2 transition state - true indicates that the transition is ongoing
     * Expected to be protected by hwaccess_lock */
    bool l2_transitioning;
};
#endif /* CONFIG_MALI_TRACE_TIMELINE */


struct kbasep_kctx_list_element {
    struct list_head link;
    struct kbase_context *kctx;
};

struct kbase_pm_device_data {
    struct mutex lock;

    int active_count;
    bool suspending;
    /* Wait queue set when active_count == 0 */
    wait_queue_head_t zero_active_count_wait;

    u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS];
    u64 debug_core_mask_all;

     int (*callback_power_runtime_init)(struct kbase_device *kbdev);

    void (*callback_power_runtime_term)(struct kbase_device *kbdev);

    /* Time in milliseconds between each dvfs sample */
    u32 dvfs_period;

    /* Period of GPU poweroff timer */
    ktime_t gpu_poweroff_time;

    /* Number of ticks of GPU poweroff timer before shader is powered off */
    int poweroff_shader_ticks;

    /* Number of ticks of GPU poweroff timer before GPU is powered off */
    int poweroff_gpu_ticks;

    struct kbase_pm_backend_data backend;
};

struct kbase_protected_ops {
    int (*protected_mode_enter)(struct kbase_device *kbdev);

    int (*protected_mode_reset)(struct kbase_device *kbdev);

    bool (*protected_mode_supported)(struct kbase_device *kbdev);
};


struct kbase_mem_pool {
    struct kbase_device *kbdev;
    size_t              cur_size;
    size_t              max_size;
    spinlock_t          pool_lock;
    struct list_head    page_list;
    struct shrinker     reclaim;

    struct kbase_mem_pool *next_pool;
};


#define DEVNAME_SIZE    16

struct kbase_device {
    s8 slot_submit_count_irq[BASE_JM_MAX_NR_SLOTS];

    u32 hw_quirks_sc;
    u32 hw_quirks_tiler;
    u32 hw_quirks_mmu;
    u32 hw_quirks_jm;

    struct list_head entry;
    struct device *dev;
    struct miscdevice mdev;
    u64 reg_start;
    size_t reg_size;
    void __iomem *reg;

    void __iomem *crgreg;
    void __iomem *pmctrlreg;
    void __iomem *pctrlreg;

    struct {
        int irq;
        int flags;
    } irqs[3];

    struct clk *clock;
#ifdef CONFIG_REGULATOR
    struct regulator *regulator;
#endif
    char devname[DEVNAME_SIZE];

#ifdef CONFIG_MALI_NO_MALI
    void *model;
    struct kmem_cache *irq_slab;
    struct workqueue_struct *irq_workq;
    atomic_t serving_job_irq;
    atomic_t serving_gpu_irq;
    atomic_t serving_mmu_irq;
    spinlock_t reg_op_lock;
#endif  /* CONFIG_MALI_NO_MALI */

    struct kbase_pm_device_data pm;
    struct kbasep_js_device_data js_data;
    struct kbase_mem_pool mem_pool;
    struct kbasep_mem_device memdev;
    struct kbase_mmu_mode const *mmu_mode;

    struct kbase_as as[BASE_MAX_NR_AS];

    spinlock_t mmu_mask_change;

    struct kbase_gpu_props gpu_props;

    unsigned long hw_issues_mask[(BASE_HW_ISSUE_END + BITS_PER_LONG - 1) / BITS_PER_LONG];
    unsigned long hw_features_mask[(BASE_HW_FEATURE_END + BITS_PER_LONG - 1) / BITS_PER_LONG];

    unsigned long hi_features_mask[2];

    /* Bitmaps of cores that are currently in use (running jobs).
     * These should be kept up to date by the job scheduler.
     *
     * pm.power_change_lock should be held when accessing these members.
     *
     * kbase_pm_check_transitions_nolock() should be called when bits are
     * cleared to update the power management system and allow transitions to
     * occur. */
    u64 shader_inuse_bitmap;

    /* Refcount for cores in use */
    u32 shader_inuse_cnt[64];

    /* Bitmaps of cores the JS needs for jobs ready to run */
    u64 shader_needed_bitmap;

    /* Refcount for cores needed */
    u32 shader_needed_cnt[64];

    u32 tiler_inuse_cnt;

    u32 tiler_needed_cnt;

    /* struct for keeping track of the disjoint information
     *
     * The state  is > 0 if the GPU is in a disjoint state. Otherwise 0
     * The count is the number of disjoint events that have occurred on the GPU
     */
    struct {
        atomic_t count;
        atomic_t state;
    } disjoint_event;

    /* Refcount for tracking users of the l2 cache, e.g. when using hardware counter instrumentation. */
    u32 l2_users_count;

    /* Bitmaps of cores that are currently available (powered up and the power policy is happy for jobs to be
     * submitted to these cores. These are updated by the power management code. The job scheduler should avoid
     * submitting new jobs to any cores that are not marked as available.
     *
     * pm.power_change_lock should be held when accessing these members.
     */
    u64 shader_available_bitmap;
    u64 tiler_available_bitmap;
    u64 l2_available_bitmap;
    u64 stack_available_bitmap;

    u64 shader_ready_bitmap;
    u64 shader_transitioning_bitmap;

    s8 nr_hw_address_spaces;              
    s8 nr_user_address_spaces;            
    /* Structure used for instrumentation and HW counters dumping */
    struct kbase_hwcnt {
        /* The lock should be used when accessing any of the following members */
        spinlock_t lock;

        struct kbase_context *kctx;
        u64 addr;

        struct kbase_instr_backend backend;
    } hwcnt;

    struct kbase_vinstr_context *vinstr_ctx;

    /*value to be written to the irq_throttle register each time an irq is served */
    atomic_t irq_throttle_cycles;

#if KBASE_TRACE_ENABLE
    spinlock_t              trace_lock;
    u16                     trace_first_out;
    u16                     trace_next_in;
    struct kbase_trace            *trace_rbuf;
#endif

    u32 reset_timeout_ms;

    struct mutex cacheclean_lock;

    /* Platform specific private data to be accessed by mali_kbase_config_xxx.c only */
    void *platform_context;

    /* List of kbase_contexts created */
    struct list_head        kctx_list;
    struct mutex            kctx_list_lock;

#ifdef CONFIG_PM_DEVFREQ
    struct devfreq_dev_profile devfreq_profile;
    struct devfreq *devfreq;
    unsigned long current_freq;
    unsigned long current_voltage;
#ifdef CONFIG_DEVFREQ_THERMAL
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
    struct devfreq_cooling_device *devfreq_cooling;
#else
    struct thermal_cooling_device *devfreq_cooling;
#endif
    struct list_head ipa_power_models;
    struct kbase_ipa_model *ipa_current_model;
    struct kbase_ipa_model *ipa_configured_model;
    struct kbase_ipa_model *ipa_fallback_model;
#endif
#endif


#ifdef CONFIG_MALI_TRACE_TIMELINE
    struct kbase_trace_kbdev_timeline timeline;
#endif

    /*
     * Control for enabling job dump on failure, set when control debugfs
     * is opened.
     */
    bool job_fault_debug;

    /* procfs entry for gpu_memory */
    struct proc_dir_entry *proc_gpu_memory_dentry;

#ifdef CONFIG_HISI_DEBUG_FS
    /* directory for debugfs entries */
    struct dentry *mali_debugfs_directory;
    /* Root directory for per context entry */
    struct dentry *debugfs_ctx_directory;

#ifdef CONFIG_MALI_DEBUG
    /* bit for each as, set if there is new data to report */
    u64 debugfs_as_read_bitmap;
#endif /* CONFIG_MALI_DEBUG */

    /* failed job dump, used for separate debug process */
    wait_queue_head_t job_fault_wq;
    wait_queue_head_t job_fault_resume_wq;
    struct workqueue_struct *job_fault_resume_workq;
    struct list_head job_fault_event_list;
    spinlock_t job_fault_event_lock;
    struct kbase_context *kctx_fault;

#if !MALI_CUSTOMER_RELEASE
    /* Per-device data for register dumping interface */
    struct {
        u16 reg_offset; /* Offset of a GPU_CONTROL register to be
                   dumped upon request */
    } regs_dump_debugfs_data;
#endif /* !MALI_CUSTOMER_RELEASE */
#endif /* CONFIG_HISI_DEBUG_FS */

    /* fbdump profiling controls set by gator */
    u32 kbase_profiling_controls[FBDUMP_CONTROL_MAX];

    struct clk *clk;

#if MALI_CUSTOMER_RELEASE == 0
    /* Number of jobs that are run before a job is forced to fail and
     * replay. May be KBASEP_FORCE_REPLAY_DISABLED, to disable forced
     * failures. */
    int force_replay_limit;
    /* Count of jobs between forced failures. Incremented on each job. A
     * job is forced to fail once this is greater than or equal to
     * force_replay_limit. */
    int force_replay_count;
    /* Core requirement for jobs to be failed and replayed. May be zero. */
    base_jd_core_req force_replay_core_req;
    /* true if force_replay_limit should be randomized. The random
     * value will be in the range of 1 - KBASEP_FORCE_REPLAY_RANDOM_LIMIT.
     */
    bool force_replay_random;
#endif

    /* Total number of created contexts */
    atomic_t ctx_num;

#ifdef CONFIG_HISI_DEBUG_FS
    /* Holds the most recent register accesses */
    struct kbase_io_history io_history;
#endif /* CONFIG_HISI_DEBUG_FS */

    /* set to 0, means not restrict outstanding */
    u32 gpu_outstanding;

    struct kbase_hwaccess_data hwaccess;

    /* Count of page/bus faults waiting for workqueues to process */
    atomic_t faults_pending;

    /* true if GPU is powered off or power off operation is in progress */
    bool poweroff_pending;


    /* defaults for new context created for this device */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0))
    bool infinite_cache_active_default;
#else
    u32 infinite_cache_active_default;
#endif
    size_t mem_pool_max_size_default;

    /* current gpu coherency mode */
    u32 current_gpu_coherency_mode;
    /* system coherency mode  */
    u32 system_coherency;
    /* Flag to track when cci snoops have been enabled on the interface */
    bool cci_snoop_enabled;

    /* SMC function IDs to call into Trusted firmware to enable/disable
     * cache snooping. Value of 0 indicates that they are not used
     */
    u32 snoop_enable_smc;
    u32 snoop_disable_smc;

    /* Protected operations */
    struct kbase_protected_ops *protected_ops;

    /*
     * true when GPU is put into protected mode
     */
    bool protected_mode;

    /*
     * true when GPU is transitioning into or out of protected mode
     */
    bool protected_mode_transition;

    /*
     * true if protected mode is supported
     */
    bool protected_mode_support;


#ifdef CONFIG_MALI_DEBUG
    wait_queue_head_t driver_inactive_wait;
    bool driver_inactive;
#endif /* CONFIG_MALI_DEBUG */

#ifdef CONFIG_MALI_FPGA_BUS_LOGGER
    /*
     * Bus logger integration.
     */
    struct bus_logger_client *buslogger;
#endif
    /* Boolean indicating if an IRQ flush during reset is in progress. */
    bool irq_reset_flush;

    /* list of inited sub systems. Used during terminate/error recovery */
    u32 inited_subsys;

    spinlock_t hwaccess_lock;

    /* Protects access to MMU operations */
    struct mutex mmu_hw_mutex;

    /* Current serialization mode. See KBASE_SERIALIZE_* for details */
    u8 serialize_jobs;
    
    /* gpu virtual id */
    u32 gpu_vid;
};

struct jsctx_queue {
    struct rb_root runnable_tree;
    struct list_head x_dep_head;
};


#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20)  | \
                     (((minor) & 0xFFF) << 8) | \
                     ((0 & 0xFF) << 0))

enum kbase_context_flags {
    KCTX_COMPAT = 1U << 0,
    KCTX_RUNNABLE_REF = 1U << 1,
    KCTX_ACTIVE = 1U << 2,
    KCTX_PULLED = 1U << 3,
    KCTX_MEM_PROFILE_INITIALIZED = 1U << 4,
    KCTX_INFINITE_CACHE = 1U << 5,
    KCTX_SUBMIT_DISABLED = 1U << 6,
    KCTX_PRIVILEGED = 1U << 7,
    KCTX_SCHEDULED = 1U << 8,
    KCTX_DYING = 1U << 9,
    KCTX_NO_IMPLICIT_SYNC = 1U << 10,
};

struct kbase_context {
    struct file *filp;
    struct kbase_device *kbdev;
    int id; /* System wide unique id */
    unsigned long api_version;
    phys_addr_t pgd;
    struct list_head event_list;
    struct list_head event_coalesce_list;
    struct mutex event_mutex;
    atomic_t event_closed;
    struct workqueue_struct *event_workq;
    atomic_t event_count;
    int event_coalesce_count;

    atomic_t flags;

    atomic_t                setup_complete;
    atomic_t                setup_in_progress;

    u64 *mmu_teardown_pages;

    struct page *aliasing_sink_page;

    struct mutex            mmu_lock;
    struct mutex            reg_lock; /* To be converted to a rwlock? */
    struct rb_root reg_rbtree_same; /* RB tree of GPU (live) regions,
                     * SAME_VA zone */
    struct rb_root reg_rbtree_exec; /* RB tree of GPU (live) regions,
                     * EXEC zone */
    struct rb_root reg_rbtree_custom; /* RB tree of GPU (live) regions,
                     * CUSTOM_VA zone */

    unsigned long    cookies;
    struct kbase_va_region *pending_regions[BITS_PER_LONG];

    wait_queue_head_t event_queue;
    pid_t tgid;
    pid_t pid;

    struct kbase_jd_context jctx;
    atomic_t used_pages;
    atomic_t         nonmapped_pages;

    struct kbase_mem_pool mem_pool;

    struct shrinker         reclaim;
    struct list_head        evict_list;

    struct list_head waiting_soft_jobs;
    spinlock_t waiting_soft_jobs_lock;
#ifdef CONFIG_KDS
    struct list_head waiting_kds_resource;
#endif
#ifdef CONFIG_MALI_DMA_FENCE
    struct {
        struct list_head waiting_resource;
        struct workqueue_struct *wq;
    } dma_fence;
#endif /* CONFIG_MALI_DMA_FENCE */

    int as_nr;

    /* NOTE:
     *
     * Flags are in jctx.sched_info.ctx.flags
     * Mutable flags *must* be accessed under jctx.sched_info.ctx.jsctx_mutex
     *
     * All other flags must be added there */
    spinlock_t         mm_update_lock;
    struct mm_struct *process_mm;
    /* End of the SAME_VA zone */
    u64 same_va_end;

#ifdef CONFIG_MALI_TRACE_TIMELINE
    struct kbase_trace_kctx_timeline timeline;
#endif
#ifdef CONFIG_HISI_DEBUG_FS
    /* Content of mem_profile file */
    char *mem_profile_data;
    /* Size of @c mem_profile_data */
    size_t mem_profile_size;
    /* Mutex guarding memory profile state */
    struct mutex mem_profile_lock;
    /* Memory profile directory under debugfs */
    struct dentry *kctx_dentry;

    /* for job fault debug */
    unsigned int *reg_dump;
    atomic_t job_fault_count;
    /* This list will keep the following atoms during the dump
     * in the same context
     */
    struct list_head job_fault_resume_event_list;

#endif /* CONFIG_HISI_DEBUG_FS */

    struct jsctx_queue jsctx_queue
        [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS];

    /* Number of atoms currently pulled from this context */
    atomic_t atoms_pulled;
    /* Number of atoms currently pulled from this context, per slot */
    atomic_t atoms_pulled_slot[BASE_JM_MAX_NR_SLOTS];
    /* Number of atoms currently pulled from this context, per slot and
     * priority. Hold hwaccess_lock when accessing */
    int atoms_pulled_slot_pri[BASE_JM_MAX_NR_SLOTS][
            KBASE_JS_ATOM_SCHED_PRIO_COUNT];

    /* true if slot is blocked on the given priority. This will be set on a
     * soft-stop */
    bool blocked_js[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT];

    /* Bitmask of slots that can be pulled from */
    u32 slots_pullable;

    /* Backend specific data */
    struct kbase_context_backend backend;

    /* Work structure used for deferred ASID assignment */
    struct work_struct work;

    /* Only one userspace vinstr client per kbase context */
    struct kbase_vinstr_client *vinstr_cli;
    struct mutex vinstr_cli_lock;

    /* List of completed jobs waiting for events to be posted */
    struct list_head completed_jobs;
    /* Number of work items currently pending on job_done_wq */
    atomic_t work_count;

    /* Waiting soft-jobs will fail when this timer expires */
    struct timer_list soft_job_timeout;

    /* JIT allocation management */
    struct kbase_va_region *jit_alloc[256];
    struct list_head jit_active_head;
    struct list_head jit_pool_head;
    struct list_head jit_destroy_head;
    struct mutex jit_evict_lock;
    struct work_struct jit_work;

    /* A list of the JIT soft-jobs in submission order
     * (protected by kbase_jd_context.lock)
     */
    struct list_head jit_atoms_head;
    /* A list of pending JIT alloc soft-jobs (using the 'queue' list_head)
     * (protected by kbase_jd_context.lock)
     */
    struct list_head jit_pending_alloc;

    /* External sticky resource management */
    struct list_head ext_res_meta_head;

    /* Used to record that a drain was requested from atomic context */
    atomic_t drain_pending;

    /* Current age count, used to determine age for newly submitted atoms */
    u32 age_count;

#if BASE_DEBUG_FENCE_TIMEOUT
    struct hrtimer fence_wait_timer;
    int timer_started;
#endif
};

struct kbase_ctx_ext_res_meta {
    struct list_head ext_res_node;
    struct kbase_mem_phy_alloc *alloc;
    u64 gpu_addr;
};

enum kbase_reg_access_type {
    REG_READ,
    REG_WRITE
};

enum kbase_share_attr_bits {
    /* (1ULL << 8) bit is reserved */
    SHARE_BOTH_BITS = (2ULL << 8),  /* inner and outer shareable coherency */
    SHARE_INNER_BITS = (3ULL << 8)  /* inner shareable coherency */
};

static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev)
{
    if ((kbdev->system_coherency == COHERENCY_ACE_LITE) ||
            (kbdev->system_coherency == COHERENCY_ACE))
        return true;

    return false;
}

/* Conversion helpers for setting up high resolution timers */
#define HR_TIMER_DELAY_MSEC(x) (ns_to_ktime(((u64)(x))*1000000U))
#define HR_TIMER_DELAY_NSEC(x) (ns_to_ktime(x))

/* Maximum number of loops polling the GPU for a cache flush before we assume it must have completed */
#define KBASE_CLEAN_CACHE_MAX_LOOPS     100000
/* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */
#define KBASE_AS_INACTIVE_MAX_LOOPS     100000

/* Maximum number of times a job can be replayed */
#define BASEP_JD_REPLAY_LIMIT 15

/* JobDescriptorHeader - taken from the architecture specifications, the layout
 * is currently identical for all GPU archs. */
struct job_descriptor_header {
    u32 exception_status;
    u32 first_incomplete_task;
    u64 fault_pointer;
    u8 job_descriptor_size : 1;
    u8 job_type : 7;
    u8 job_barrier : 1;
    u8 _reserved_01 : 1;
    u8 _reserved_1 : 1;
    u8 _reserved_02 : 1;
    u8 _reserved_03 : 1;
    u8 _reserved_2 : 1;
    u8 _reserved_04 : 1;
    u8 _reserved_05 : 1;
    u16 job_index;
    u16 job_dependency_index_1;
    u16 job_dependency_index_2;
    union {
        u64 _64;
        u32 _32;
    } next_job;
};

#endif              /* _KBASE_DEFS_H_ */