/* * SPDX-License-Identifier: MIT * * Copyright © 2019 Intel Corporation */ #ifndef __INTEL_ENGINE_TYPES__ #define __INTEL_ENGINE_TYPES__ #include #include #include #include #include #include "i915_gem.h" #include "i915_priolist_types.h" #include "i915_selftest.h" #include "i915_timeline_types.h" #include "intel_workarounds_types.h" #include "i915_gem_batch_pool.h" #include "i915_pmu.h" #define I915_MAX_SLICES 3 #define I915_MAX_SUBSLICES 8 #define I915_CMD_HASH_ORDER 9 struct dma_fence; struct drm_i915_reg_table; struct i915_gem_context; struct i915_request; struct i915_sched_attr; struct intel_uncore; typedef u8 intel_engine_mask_t; #define ALL_ENGINES ((intel_engine_mask_t)~0ul) struct intel_hw_status_page { struct i915_vma *vma; u32 *addr; }; struct intel_instdone { u32 instdone; /* The following exist only in the RCS engine */ u32 slice_common; u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES]; u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES]; }; struct intel_engine_hangcheck { u64 acthd; u32 last_seqno; u32 next_seqno; unsigned long action_timestamp; struct intel_instdone instdone; }; struct intel_ring { struct kref ref; struct i915_vma *vma; void *vaddr; struct i915_timeline *timeline; struct list_head request_list; struct list_head active_link; u32 head; u32 tail; u32 emit; u32 space; u32 size; u32 effective_size; }; /* * we use a single page to load ctx workarounds so all of these * values are referred in terms of dwords * * struct i915_wa_ctx_bb: * offset: specifies batch starting position, also helpful in case * if we want to have multiple batches at different offsets based on * some criteria. It is not a requirement at the moment but provides * an option for future use. * size: size of the batch in DWORDS */ struct i915_ctx_workarounds { struct i915_wa_ctx_bb { u32 offset; u32 size; } indirect_ctx, per_ctx; struct i915_vma *vma; }; #define I915_MAX_VCS 4 #define I915_MAX_VECS 2 /* * Engine IDs definitions. * Keep instances of the same type engine together. */ enum intel_engine_id { RCS0 = 0, BCS0, VCS0, VCS1, VCS2, VCS3, #define _VCS(n) (VCS0 + (n)) VECS0, VECS1, #define _VECS(n) (VECS0 + (n)) I915_NUM_ENGINES }; struct st_preempt_hang { struct completion completion; unsigned int count; bool inject_hang; }; /** * struct intel_engine_execlists - execlist submission queue and port state * * The struct intel_engine_execlists represents the combined logical state of * driver and the hardware state for execlist mode of submission. */ struct intel_engine_execlists { /** * @tasklet: softirq tasklet for bottom handler */ struct tasklet_struct tasklet; /** * @default_priolist: priority list for I915_PRIORITY_NORMAL */ struct i915_priolist default_priolist; /** * @no_priolist: priority lists disabled */ bool no_priolist; /** * @submit_reg: gen-specific execlist submission register * set to the ExecList Submission Port (elsp) register pre-Gen11 and to * the ExecList Submission Queue Contents register array for Gen11+ */ u32 __iomem *submit_reg; /** * @ctrl_reg: the enhanced execlists control register, used to load the * submit queue on the HW and to request preemptions to idle */ u32 __iomem *ctrl_reg; /** * @port: execlist port states * * For each hardware ELSP (ExecList Submission Port) we keep * track of the last request and the number of times we submitted * that port to hw. We then count the number of times the hw reports * a context completion or preemption. As only one context can * be active on hw, we limit resubmission of context to port[0]. This * is called Lite Restore, of the context. */ struct execlist_port { /** * @request_count: combined request and submission count */ struct i915_request *request_count; #define EXECLIST_COUNT_BITS 2 #define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS) #define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS) #define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS) #define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS) #define port_set(p, packed) ((p)->request_count = (packed)) #define port_isset(p) ((p)->request_count) #define port_index(p, execlists) ((p) - (execlists)->port) /** * @context_id: context ID for port */ GEM_DEBUG_DECL(u32 context_id); #define EXECLIST_MAX_PORTS 2 } port[EXECLIST_MAX_PORTS]; /** * @active: is the HW active? We consider the HW as active after * submitting any context for execution and until we have seen the * last context completion event. After that, we do not expect any * more events until we submit, and so can park the HW. * * As we have a small number of different sources from which we feed * the HW, we track the state of each inside a single bitfield. */ unsigned int active; #define EXECLISTS_ACTIVE_USER 0 #define EXECLISTS_ACTIVE_PREEMPT 1 #define EXECLISTS_ACTIVE_HWACK 2 /** * @port_mask: number of execlist ports - 1 */ unsigned int port_mask; /** * @queue_priority_hint: Highest pending priority. * * When we add requests into the queue, or adjust the priority of * executing requests, we compute the maximum priority of those * pending requests. We can then use this value to determine if * we need to preempt the executing requests to service the queue. * However, since the we may have recorded the priority of an inflight * request we wanted to preempt but since completed, at the time of * dequeuing the priority hint may no longer may match the highest * available request priority. */ int queue_priority_hint; /** * @queue: queue of requests, in priority lists */ struct rb_root_cached queue; /** * @csb_write: control register for Context Switch buffer * * Note this register may be either mmio or HWSP shadow. */ u32 *csb_write; /** * @csb_status: status array for Context Switch buffer * * Note these register may be either mmio or HWSP shadow. */ u32 *csb_status; /** * @preempt_complete_status: expected CSB upon completing preemption */ u32 preempt_complete_status; /** * @csb_size: context status buffer FIFO size */ u8 csb_size; /** * @csb_head: context status buffer head */ u8 csb_head; I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;) }; #define INTEL_ENGINE_CS_MAX_NAME 8 struct intel_engine_cs { struct drm_i915_private *i915; struct intel_uncore *uncore; char name[INTEL_ENGINE_CS_MAX_NAME]; enum intel_engine_id id; unsigned int hw_id; unsigned int guc_id; intel_engine_mask_t mask; u8 uabi_class; u8 class; u8 instance; u32 context_size; u32 mmio_base; struct intel_ring *buffer; struct i915_timeline timeline; struct intel_context *kernel_context; /* pinned */ struct intel_context *preempt_context; /* pinned; optional */ struct drm_i915_gem_object *default_state; void *pinned_default_state; /* Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the * heavyweight seqno dance, we delegate the task (of being the * bottom-half of the user interrupt) to the first client. After * every interrupt, we wake up one client, who does the heavyweight * coherent seqno read and either goes back to sleep (if incomplete), * or wakes up all the completed clients in parallel, before then * transferring the bottom-half status to the next client in the queue. * * Compared to walking the entire list of waiters in a single dedicated * bottom-half, we reduce the latency of the first waiter by avoiding * a context switch, but incur additional coherent seqno reads when * following the chain of request breadcrumbs. Since it is most likely * that we have a single client waiting on each seqno, then reducing * the overhead of waking that client is much preferred. */ struct intel_breadcrumbs { spinlock_t irq_lock; struct list_head signalers; struct irq_work irq_work; /* for use from inside irq_lock */ unsigned int irq_enabled; bool irq_armed; } breadcrumbs; struct intel_engine_pmu { /** * @enable: Bitmask of enable sample events on this engine. * * Bits correspond to sample event types, for instance * I915_SAMPLE_QUEUED is bit 0 etc. */ u32 enable; /** * @enable_count: Reference count for the enabled samplers. * * Index number corresponds to @enum drm_i915_pmu_engine_sample. */ unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT]; /** * @sample: Counter values for sampling events. * * Our internal timer stores the current counters in this field. * * Index number corresponds to @enum drm_i915_pmu_engine_sample. */ struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT]; } pmu; /* * A pool of objects to use as shadow copies of client batch buffers * when the command parser is enabled. Prevents the client from * modifying the batch contents after software parsing. */ struct i915_gem_batch_pool batch_pool; struct intel_hw_status_page status_page; struct i915_ctx_workarounds wa_ctx; struct i915_wa_list ctx_wa_list; struct i915_wa_list wa_list; struct i915_wa_list whitelist; u32 irq_keep_mask; /* always keep these interrupts */ u32 irq_enable_mask; /* bitmask to enable ring interrupt */ void (*irq_enable)(struct intel_engine_cs *engine); void (*irq_disable)(struct intel_engine_cs *engine); int (*init_hw)(struct intel_engine_cs *engine); struct { void (*prepare)(struct intel_engine_cs *engine); void (*reset)(struct intel_engine_cs *engine, bool stalled); void (*finish)(struct intel_engine_cs *engine); } reset; void (*park)(struct intel_engine_cs *engine); void (*unpark)(struct intel_engine_cs *engine); void (*set_default_submission)(struct intel_engine_cs *engine); const struct intel_context_ops *cops; int (*request_alloc)(struct i915_request *rq); int (*init_context)(struct i915_request *rq); int (*emit_flush)(struct i915_request *request, u32 mode); #define EMIT_INVALIDATE BIT(0) #define EMIT_FLUSH BIT(1) #define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH) int (*emit_bb_start)(struct i915_request *rq, u64 offset, u32 length, unsigned int dispatch_flags); #define I915_DISPATCH_SECURE BIT(0) #define I915_DISPATCH_PINNED BIT(1) int (*emit_init_breadcrumb)(struct i915_request *rq); u32 *(*emit_fini_breadcrumb)(struct i915_request *rq, u32 *cs); unsigned int emit_fini_breadcrumb_dw; /* Pass the request to the hardware queue (e.g. directly into * the legacy ringbuffer or to the end of an execlist). * * This is called from an atomic context with irqs disabled; must * be irq safe. */ void (*submit_request)(struct i915_request *rq); /* * Call when the priority on a request has changed and it and its * dependencies may need rescheduling. Note the request itself may * not be ready to run! */ void (*schedule)(struct i915_request *request, const struct i915_sched_attr *attr); /* * Cancel all requests on the hardware, or queued for execution. * This should only cancel the ready requests that have been * submitted to the engine (via the engine->submit_request callback). * This is called when marking the device as wedged. */ void (*cancel_requests)(struct intel_engine_cs *engine); void (*cleanup)(struct intel_engine_cs *engine); struct intel_engine_execlists execlists; /* Contexts are pinned whilst they are active on the GPU. The last * context executed remains active whilst the GPU is idle - the * switch away and write to the context object only occurs on the * next execution. Contexts are only unpinned on retirement of the * following request ensuring that we can always write to the object * on the context switch even after idling. Across suspend, we switch * to the kernel context and trash it as the save may not happen * before the hardware is powered down. */ struct intel_context *last_retired_context; /* status_notifier: list of callbacks for context-switch changes */ struct atomic_notifier_head context_status_notifier; struct intel_engine_hangcheck hangcheck; #define I915_ENGINE_NEEDS_CMD_PARSER BIT(0) #define I915_ENGINE_SUPPORTS_STATS BIT(1) #define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_SEMAPHORES BIT(3) #define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4) unsigned int flags; /* * Table of commands the command parser needs to know about * for this engine. */ DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER); /* * Table of registers allowed in commands that read/write registers. */ const struct drm_i915_reg_table *reg_tables; int reg_table_count; /* * Returns the bitmask for the length field of the specified command. * Return 0 for an unrecognized/invalid command. * * If the command parser finds an entry for a command in the engine's * cmd_tables, it gets the command's length based on the table entry. * If not, it calls this function to determine the per-engine length * field encoding for the command (i.e. different opcode ranges use * certain bits to encode the command length in the header). */ u32 (*get_cmd_length_mask)(u32 cmd_header); struct { /** * @lock: Lock protecting the below fields. */ seqlock_t lock; /** * @enabled: Reference count indicating number of listeners. */ unsigned int enabled; /** * @active: Number of contexts currently scheduled in. */ unsigned int active; /** * @enabled_at: Timestamp when busy stats were enabled. */ ktime_t enabled_at; /** * @start: Timestamp of the last idle to active transition. * * Idle is defined as active == 0, active is active > 0. */ ktime_t start; /** * @total: Total time this engine was busy. * * Accumulated time not counting the most recent block in cases * where engine is currently busy (active > 0). */ ktime_t total; } stats; }; static inline bool intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER; } static inline bool intel_engine_supports_stats(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_SUPPORTS_STATS; } static inline bool intel_engine_has_preemption(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_HAS_PREEMPTION; } static inline bool intel_engine_has_semaphores(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_HAS_SEMAPHORES; } static inline bool intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; } #define instdone_slice_mask(dev_priv__) \ (IS_GEN(dev_priv__, 7) ? \ 1 : RUNTIME_INFO(dev_priv__)->sseu.slice_mask) #define instdone_subslice_mask(dev_priv__) \ (IS_GEN(dev_priv__, 7) ? \ 1 : RUNTIME_INFO(dev_priv__)->sseu.subslice_mask[0]) #define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \ for ((slice__) = 0, (subslice__) = 0; \ (slice__) < I915_MAX_SLICES; \ (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \ (slice__) += ((subslice__) == 0)) \ for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \ (BIT(subslice__) & instdone_subslice_mask(dev_priv__))) #endif /* __INTEL_ENGINE_TYPES_H__ */