diff options
Diffstat (limited to 'drivers/iommu/apple-dart.c')
| -rw-r--r-- | drivers/iommu/apple-dart.c | 672 |
1 files changed, 546 insertions, 126 deletions
diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 4f4a323be0d0..f8648dcea646 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -33,58 +33,157 @@ #include <linux/types.h> #include "dma-iommu.h" +#include "io-pgtable-dart.h" -#define DART_MAX_STREAMS 16 +#define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 2 -#define DART_STREAM_ALL 0xffff +/* Common registers */ #define DART_PARAMS1 0x00 -#define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24) +#define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) #define DART_PARAMS2 0x04 -#define DART_PARAMS_BYPASS_SUPPORT BIT(0) +#define DART_PARAMS2_BYPASS_SUPPORT BIT(0) -#define DART_STREAM_COMMAND 0x20 -#define DART_STREAM_COMMAND_BUSY BIT(2) -#define DART_STREAM_COMMAND_INVALIDATE BIT(20) +/* T8020/T6000 registers */ -#define DART_STREAM_SELECT 0x34 +#define DART_T8020_STREAM_COMMAND 0x20 +#define DART_T8020_STREAM_COMMAND_BUSY BIT(2) +#define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) -#define DART_ERROR 0x40 -#define DART_ERROR_STREAM GENMASK(27, 24) -#define DART_ERROR_CODE GENMASK(11, 0) -#define DART_ERROR_FLAG BIT(31) +#define DART_T8020_STREAM_SELECT 0x34 -#define DART_ERROR_READ_FAULT BIT(4) -#define DART_ERROR_WRITE_FAULT BIT(3) -#define DART_ERROR_NO_PTE BIT(2) -#define DART_ERROR_NO_PMD BIT(1) -#define DART_ERROR_NO_TTBR BIT(0) +#define DART_T8020_ERROR 0x40 +#define DART_T8020_ERROR_STREAM GENMASK(27, 24) +#define DART_T8020_ERROR_CODE GENMASK(11, 0) +#define DART_T8020_ERROR_FLAG BIT(31) -#define DART_CONFIG 0x60 -#define DART_CONFIG_LOCK BIT(15) +#define DART_T8020_ERROR_READ_FAULT BIT(4) +#define DART_T8020_ERROR_WRITE_FAULT BIT(3) +#define DART_T8020_ERROR_NO_PTE BIT(2) +#define DART_T8020_ERROR_NO_PMD BIT(1) +#define DART_T8020_ERROR_NO_TTBR BIT(0) -#define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 - -#define DART_ERROR_ADDR_HI 0x54 -#define DART_ERROR_ADDR_LO 0x50 - -#define DART_STREAMS_ENABLE 0xfc +#define DART_T8020_CONFIG 0x60 +#define DART_T8020_CONFIG_LOCK BIT(15) -#define DART_TCR(sid) (0x100 + 4 * (sid)) -#define DART_TCR_TRANSLATE_ENABLE BIT(7) -#define DART_TCR_BYPASS0_ENABLE BIT(8) -#define DART_TCR_BYPASS1_ENABLE BIT(12) +#define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 -#define DART_TTBR(sid, idx) (0x200 + 16 * (sid) + 4 * (idx)) -#define DART_TTBR_VALID BIT(31) -#define DART_TTBR_SHIFT 12 +#define DART_T8020_ERROR_ADDR_HI 0x54 +#define DART_T8020_ERROR_ADDR_LO 0x50 + +#define DART_T8020_STREAMS_ENABLE 0xfc + +#define DART_T8020_TCR 0x100 +#define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) +#define DART_T8020_TCR_BYPASS_DART BIT(8) +#define DART_T8020_TCR_BYPASS_DAPF BIT(12) + +#define DART_T8020_TTBR 0x200 +#define DART_T8020_TTBR_VALID BIT(31) +#define DART_T8020_TTBR_ADDR_OFF 0 +#define DART_T8020_TTBR_SHIFT 12 + +/* T8110 registers */ + +#define DART_T8110_PARAMS3 0x08 +#define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) +#define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) +#define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) +#define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) + +#define DART_T8110_PARAMS4 0x0c +#define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) +#define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) + +#define DART_T8110_TLB_CMD 0x80 +#define DART_T8110_TLB_CMD_BUSY BIT(31) +#define DART_T8110_TLB_CMD_OP GENMASK(10, 8) +#define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 +#define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 +#define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) + +#define DART_T8110_ERROR 0x100 +#define DART_T8110_ERROR_STREAM GENMASK(27, 20) +#define DART_T8110_ERROR_CODE GENMASK(14, 0) +#define DART_T8110_ERROR_FLAG BIT(31) + +#define DART_T8110_ERROR_MASK 0x104 + +#define DART_T8110_ERROR_READ_FAULT BIT(4) +#define DART_T8110_ERROR_WRITE_FAULT BIT(3) +#define DART_T8110_ERROR_NO_PTE BIT(3) +#define DART_T8110_ERROR_NO_PMD BIT(2) +#define DART_T8110_ERROR_NO_PGD BIT(1) +#define DART_T8110_ERROR_NO_TTBR BIT(0) + +#define DART_T8110_ERROR_ADDR_LO 0x170 +#define DART_T8110_ERROR_ADDR_HI 0x174 + +#define DART_T8110_PROTECT 0x200 +#define DART_T8110_UNPROTECT 0x204 +#define DART_T8110_PROTECT_LOCK 0x208 +#define DART_T8110_PROTECT_TTBR_TCR BIT(0) + +#define DART_T8110_ENABLE_STREAMS 0xc00 +#define DART_T8110_DISABLE_STREAMS 0xc20 + +#define DART_T8110_TCR 0x1000 +#define DART_T8110_TCR_REMAP GENMASK(11, 8) +#define DART_T8110_TCR_REMAP_EN BIT(7) +#define DART_T8110_TCR_BYPASS_DAPF BIT(2) +#define DART_T8110_TCR_BYPASS_DART BIT(1) +#define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) + +#define DART_T8110_TTBR 0x1400 +#define DART_T8110_TTBR_VALID BIT(0) +#define DART_T8110_TTBR_ADDR_OFF 2 +#define DART_T8110_TTBR_SHIFT 14 + +#define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) + +#define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ + (((dart)->hw->ttbr_count * (sid)) << 2) + \ + ((idx) << 2)) + +struct apple_dart_stream_map; + +enum dart_type { + DART_T8020, + DART_T6000, + DART_T8110, +}; struct apple_dart_hw { + enum dart_type type; + irqreturn_t (*irq_handler)(int irq, void *dev); + int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); + u32 oas; enum io_pgtable_fmt fmt; + + int max_sid_count; + + u64 lock; + u64 lock_bit; + + u64 error; + + u64 enable_streams; + u64 disable_streams; + + u64 tcr; + u64 tcr_enabled; + u64 tcr_disabled; + u64 tcr_bypass; + + u64 ttbr; + u64 ttbr_valid; + u64 ttbr_addr_off; + u64 ttbr_shift; + int ttbr_count; }; /* @@ -100,6 +199,7 @@ struct apple_dart_hw { * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode * @force_bypass: force bypass mode due to pagesize mismatch? + * @locked: indicates if this DART is locked * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ @@ -115,12 +215,18 @@ struct apple_dart { spinlock_t lock; + u32 oas; u32 pgsize; + u32 num_streams; u32 supports_bypass : 1; u32 force_bypass : 1; + u32 locked : 1; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; + + u32 save_tcr[DART_MAX_STREAMS]; + u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -140,11 +246,11 @@ struct apple_dart { */ struct apple_dart_stream_map { struct apple_dart *dart; - unsigned long sidmap; + DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); }; struct apple_dart_atomic_stream_map { struct apple_dart *dart; - atomic64_t sidmap; + atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; }; /* @@ -202,50 +308,60 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) static void apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(DART_TCR_TRANSLATE_ENABLE, - stream_map->dart->regs + DART_TCR(sid)); + WARN_ON(stream_map->dart->locked); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); } static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(0, stream_map->dart->regs + DART_TCR(sid)); + WARN_ON(stream_map->dart->locked); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); } static void apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; + WARN_ON(stream_map->dart->locked); WARN_ON(!stream_map->dart->supports_bypass); - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE, - stream_map->dart->regs + DART_TCR(sid)); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(dart->hw->tcr_bypass, + dart->regs + DART_TCR(dart, sid)); } static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, u8 idx, phys_addr_t paddr) { + struct apple_dart *dart = stream_map->dart; int sid; - WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1)); - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT), - stream_map->dart->regs + DART_TTBR(sid, idx)); + WARN_ON(stream_map->dart->locked); + WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(dart->hw->ttbr_valid | + (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_off, + dart->regs + DART_TTBR(dart, sid, idx)); } static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(0, stream_map->dart->regs + DART_TTBR(sid, idx)); + WARN_ON(stream_map->dart->locked); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(0, dart->regs + DART_TTBR(dart, sid, idx)); } static void @@ -253,12 +369,12 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) { int i; - for (i = 0; i < DART_MAX_TTBR; ++i) + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) apple_dart_hw_clear_ttbr(stream_map, i); } static int -apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, +apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) { unsigned long flags; @@ -267,12 +383,12 @@ apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, spin_lock_irqsave(&stream_map->dart->lock, flags); - writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT); - writel(command, stream_map->dart->regs + DART_STREAM_COMMAND); + writel(stream_map->sidmap[0], stream_map->dart->regs + DART_T8020_STREAM_SELECT); + writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); ret = readl_poll_timeout_atomic( - stream_map->dart->regs + DART_STREAM_COMMAND, command_reg, - !(command_reg & DART_STREAM_COMMAND_BUSY), 1, + stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, + !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, DART_STREAM_COMMAND_BUSY_TIMEOUT); spin_unlock_irqrestore(&stream_map->dart->lock, flags); @@ -280,7 +396,45 @@ apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, if (ret) { dev_err(stream_map->dart->dev, "busy bit did not clear after command %x for streams %lx\n", - command, stream_map->sidmap); + command, stream_map->sidmap[0]); + return ret; + } + + return 0; +} + +static int +apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, + u32 command) +{ + struct apple_dart *dart = stream_map->dart; + unsigned long flags; + int ret = 0; + int sid; + + spin_lock_irqsave(&dart->lock, flags); + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | + FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); + writel(val, dart->regs + DART_T8110_TLB_CMD); + + ret = readl_poll_timeout_atomic( + dart->regs + DART_T8110_TLB_CMD, val, + !(val & DART_T8110_TLB_CMD_BUSY), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); + + if (ret) + break; + + } + + spin_unlock_irqrestore(&dart->lock, flags); + + if (ret) { + dev_err(stream_map->dart->dev, + "busy bit did not clear after command %x for stream %d\n", + command, sid); return ret; } @@ -288,48 +442,56 @@ apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, } static int -apple_dart_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) +apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { - return apple_dart_hw_stream_command(stream_map, - DART_STREAM_COMMAND_INVALIDATE); + return apple_dart_t8020_hw_stream_command( + stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); +} + +static int +apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) +{ + return apple_dart_t8110_hw_tlb_command( + stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); } static int apple_dart_hw_reset(struct apple_dart *dart) { - u32 config; struct apple_dart_stream_map stream_map; - - config = readl(dart->regs + DART_CONFIG); - if (config & DART_CONFIG_LOCK) { - dev_err(dart->dev, "DART is locked down until reboot: %08x\n", - config); - return -EINVAL; - } + int i; stream_map.dart = dart; - stream_map.sidmap = DART_STREAM_ALL; + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); + bitmap_set(stream_map.sidmap, 0, dart->num_streams); apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); /* enable all streams globally since TCR is used to control isolation */ - writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + writel(U32_MAX, dart->regs + dart->hw->enable_streams); /* clear any pending errors before the interrupt is unmasked */ - writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR); + writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); - return apple_dart_hw_invalidate_tlb(&stream_map); + if (dart->hw->type == DART_T8110) + writel(0, dart->regs + DART_T8110_ERROR_MASK); + + return dart->hw->invalidate_tlb(&stream_map); } static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) { - int i; + int i, j; struct apple_dart_atomic_stream_map *domain_stream_map; struct apple_dart_stream_map stream_map; for_each_stream_map(i, domain, domain_stream_map) { stream_map.dart = domain_stream_map->dart; - stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap); - apple_dart_hw_invalidate_tlb(&stream_map); + + for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) + stream_map.sidmap[j] = atomic64_read(&domain_stream_map->sidmap[j]); + + stream_map.dart->hw->invalidate_tlb(&stream_map); } } @@ -396,24 +558,62 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_set_ttbr(stream_map, i, - pgtbl_cfg->apple_dart_cfg.ttbr[i]); - for (; i < DART_MAX_TTBR; ++i) - apple_dart_hw_clear_ttbr(stream_map, i); + /* Locked DARTs are set up by the bootloader. */ + if (!stream_map->dart->locked) { + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) + apple_dart_hw_set_ttbr(stream_map, i, + pgtbl_cfg->apple_dart_cfg.ttbr[i]); + for (; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_clear_ttbr(stream_map, i); - apple_dart_hw_enable_translation(stream_map); - apple_dart_hw_invalidate_tlb(stream_map); + apple_dart_hw_enable_translation(stream_map); + } + stream_map->dart->hw->invalidate_tlb(stream_map); +} + +static int apple_dart_setup_resv_locked(struct iommu_domain *domain, + struct device *dev, size_t pgsize) +{ + struct iommu_resv_region *region; + LIST_HEAD(resv_regions); + int ret = 0; + + of_iommu_get_resv_regions(dev, &resv_regions); + list_for_each_entry(region, &resv_regions, list) { + size_t mapped = 0; + + /* Only map translated reserved regions */ + if (region->type != IOMMU_RESV_TRANSLATED) + continue; + + while (mapped < region->length) { + phys_addr_t paddr = region->start + mapped; + unsigned long iova = region->dva + mapped; + size_t length = region->length - mapped; + size_t pgcount = length / pgsize; + + ret = apple_dart_map_pages(domain, iova, + paddr, pgsize, pgcount, + region->prot, GFP_KERNEL, &mapped); + + if (ret) + goto end_put; + } + } +end_put: + iommu_put_resv_regions(dev, &resv_regions); + return ret; } static int apple_dart_finalize_domain(struct iommu_domain *domain, + struct device *dev, struct apple_dart_master_cfg *cfg) { struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; int ret = 0; - int i; + int i, j; mutex_lock(&dart_domain->init_lock); @@ -422,18 +622,40 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; - atomic64_set(&dart_domain->stream_maps[i].sidmap, - cfg->stream_maps[i].sidmap); + for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) + atomic64_set(&dart_domain->stream_maps[i].sidmap[j], + cfg->stream_maps[i].sidmap[j]); } pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, .ias = 32, - .oas = dart->hw->oas, + .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, }; + if (dart->locked) { + unsigned long *sidmap; + int sid; + phys_addr_t phys; + u32 ttbr; + + /* Locked DARTs can only have a single stream bound */ + sidmap = cfg->stream_maps[0].sidmap; + sid = find_first_bit(sidmap, dart->num_streams); + + WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + + WARN_ON(!(ttbr & dart->hw->ttbr_valid)); + ttbr &= ~dart->hw->ttbr_valid; + + phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; + pgtbl_cfg.apple_dart_cfg.ttbr[0] = phys; + pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_APPLE_LOCKED; + } + dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain); if (!dart_domain->pgtbl_ops) { @@ -448,6 +670,11 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, dart_domain->finalized = true; + if (dart->locked) { + /* TODO: error handling */ + ret = apple_dart_setup_resv_locked(domain, dev, dart->pgsize); + io_pgtable_dart_setup_locked(dart_domain->pgtbl_ops); + } done: mutex_unlock(&dart_domain->init_lock); return ret; @@ -458,7 +685,7 @@ apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, struct apple_dart_stream_map *master_maps, bool add_streams) { - int i; + int i, j; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (domain_maps[i].dart != master_maps[i].dart) @@ -468,12 +695,14 @@ apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (!domain_maps[i].dart) break; - if (add_streams) - atomic64_or(master_maps[i].sidmap, - &domain_maps[i].sidmap); - else - atomic64_and(~master_maps[i].sidmap, - &domain_maps[i].sidmap); + for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { + if (add_streams) + atomic64_or(master_maps[i].sidmap[j], + &domain_maps[i].sidmap[j]); + else + atomic64_and(~master_maps[i].sidmap[j], + &domain_maps[i].sidmap[j]); + } } return 0; @@ -500,15 +729,16 @@ static int apple_dart_attach_dev(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + struct apple_dart *dart0 = cfg->stream_maps[0].dart; - if (cfg->stream_maps[0].dart->force_bypass && - domain->type != IOMMU_DOMAIN_IDENTITY) + if (dart0->force_bypass && domain->type != IOMMU_DOMAIN_IDENTITY) return -EINVAL; - if (!cfg->stream_maps[0].dart->supports_bypass && - domain->type == IOMMU_DOMAIN_IDENTITY) + if (!dart0->supports_bypass && domain->type == IOMMU_DOMAIN_IDENTITY) + return -EINVAL; + if (dart0->locked && domain->type != IOMMU_DOMAIN_DMA) return -EINVAL; - ret = apple_dart_finalize_domain(domain, cfg); + ret = apple_dart_finalize_domain(domain, dev, cfg); if (ret) return ret; @@ -637,14 +867,14 @@ static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (cfg->stream_maps[i].dart == dart) { - cfg->stream_maps[i].sidmap |= 1 << sid; + set_bit(sid, cfg->stream_maps[i].sidmap); return 0; } } for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (!cfg->stream_maps[i].dart) { cfg->stream_maps[i].dart = dart; - cfg->stream_maps[i].sidmap = 1 << sid; + set_bit(sid, cfg->stream_maps[i].sidmap); return 0; } } @@ -663,7 +893,7 @@ static void apple_dart_release_group(void *iommu_data) mutex_lock(&apple_dart_groups_lock); for_each_stream_map(i, group_master_cfg, stream_map) - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) stream_map->dart->sid2group[sid] = NULL; kfree(iommu_data); @@ -682,7 +912,7 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) mutex_lock(&apple_dart_groups_lock); for_each_stream_map(i, cfg, stream_map) { - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) { + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { struct iommu_group *stream_group = stream_map->dart->sid2group[sid]; @@ -721,7 +951,7 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) apple_dart_release_group); for_each_stream_map(i, cfg, stream_map) - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) stream_map->dart->sid2group[sid] = group; res = group; @@ -734,10 +964,15 @@ out: static int apple_dart_def_domain_type(struct device *dev) { struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + struct apple_dart *dart = cfg->stream_maps[0].dart; - if (cfg->stream_maps[0].dart->force_bypass) + WARN_ON(dart->force_bypass && dart->locked); + + if (dart->force_bypass) return IOMMU_DOMAIN_IDENTITY; - if (!cfg->stream_maps[0].dart->supports_bypass) + if (dart->locked) + return IOMMU_DOMAIN_DMA; + if (dart->supports_bypass) return IOMMU_DOMAIN_DMA; return 0; @@ -791,30 +1026,30 @@ static const struct iommu_ops apple_dart_iommu_ops = { } }; -static irqreturn_t apple_dart_irq(int irq, void *dev) +static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) { struct apple_dart *dart = dev; const char *fault_name = NULL; - u32 error = readl(dart->regs + DART_ERROR); - u32 error_code = FIELD_GET(DART_ERROR_CODE, error); - u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO); - u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI); + u32 error = readl(dart->regs + DART_T8020_ERROR); + u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); + u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); + u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); u64 addr = addr_lo | (((u64)addr_hi) << 32); - u8 stream_idx = FIELD_GET(DART_ERROR_STREAM, error); + u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); - if (!(error & DART_ERROR_FLAG)) + if (!(error & DART_T8020_ERROR_FLAG)) return IRQ_NONE; /* there should only be a single bit set but let's use == to be sure */ - if (error_code == DART_ERROR_READ_FAULT) + if (error_code == DART_T8020_ERROR_READ_FAULT) fault_name = "READ FAULT"; - else if (error_code == DART_ERROR_WRITE_FAULT) + else if (error_code == DART_T8020_ERROR_WRITE_FAULT) fault_name = "WRITE FAULT"; - else if (error_code == DART_ERROR_NO_PTE) + else if (error_code == DART_T8020_ERROR_NO_PTE) fault_name = "NO PTE FOR IOVA"; - else if (error_code == DART_ERROR_NO_PMD) + else if (error_code == DART_T8020_ERROR_NO_PMD) fault_name = "NO PMD FOR IOVA"; - else if (error_code == DART_ERROR_NO_TTBR) + else if (error_code == DART_T8020_ERROR_NO_TTBR) fault_name = "NO TTBR FOR IOVA"; else fault_name = "unknown"; @@ -824,14 +1059,58 @@ static irqreturn_t apple_dart_irq(int irq, void *dev) "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", error, stream_idx, error_code, fault_name, addr); - writel(error, dart->regs + DART_ERROR); + writel(error, dart->regs + DART_T8020_ERROR); return IRQ_HANDLED; } +static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) +{ + struct apple_dart *dart = dev; + const char *fault_name = NULL; + u32 error = readl(dart->regs + DART_T8110_ERROR); + u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); + u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); + u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); + u64 addr = addr_lo | (((u64)addr_hi) << 32); + u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); + + if (!(error & DART_T8110_ERROR_FLAG)) + return IRQ_NONE; + + /* there should only be a single bit set but let's use == to be sure */ + if (error_code == DART_T8110_ERROR_READ_FAULT) + fault_name = "READ FAULT"; + else if (error_code == DART_T8110_ERROR_WRITE_FAULT) + fault_name = "WRITE FAULT"; + else if (error_code == DART_T8110_ERROR_NO_PTE) + fault_name = "NO PTE FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_PMD) + fault_name = "NO PMD FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_PGD) + fault_name = "NO PGD FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_TTBR) + fault_name = "NO TTBR FOR IOVA"; + else + fault_name = "unknown"; + + dev_err_ratelimited( + dart->dev, + "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", + error, stream_idx, error_code, fault_name, addr); + + writel(error, dart->regs + DART_T8110_ERROR); + return IRQ_HANDLED; +} + +static bool apple_dart_is_locked(struct apple_dart *dart) +{ + return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret; - u32 dart_params[2]; + u32 dart_params[4]; struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; @@ -866,17 +1145,42 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; - dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); - dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]); - dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT; + dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); + dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; + + switch (dart->hw->type) { + case DART_T8020: + case DART_T6000: + dart->oas = dart->hw->oas; + dart->num_streams = dart->hw->max_sid_count; + break; + + case DART_T8110: + dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); + dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); + dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); + dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); + break; + } + + if (dart->num_streams > DART_MAX_STREAMS) { + dev_err(&pdev->dev, "Too many streams (%d > %d)\n", + dart->num_streams, DART_MAX_STREAMS); + goto err_clk_disable; + } + dart->force_bypass = dart->pgsize > PAGE_SIZE; - ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, + dart->locked = apple_dart_is_locked(dart); + if (!dart->locked) { + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + } + + ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; @@ -894,8 +1198,8 @@ static int apple_dart_probe(struct platform_device *pdev) dev_info( &pdev->dev, - "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n", - dart->pgsize, dart->supports_bypass, dart->force_bypass); + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d] initialized\n", + dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass, dart->locked); return 0; err_sysfs_remove: @@ -924,15 +1228,128 @@ static int apple_dart_remove(struct platform_device *pdev) } static const struct apple_dart_hw apple_dart_hw_t8103 = { + .type = DART_T8020, + .irq_handler = apple_dart_t8020_irq, + .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, .oas = 36, .fmt = APPLE_DART, + .max_sid_count = 16, + + .enable_streams = DART_T8020_STREAMS_ENABLE, + .lock = DART_T8020_CONFIG, + .lock_bit = DART_T8020_CONFIG_LOCK, + + .error = DART_T8020_ERROR, + + .tcr = DART_T8020_TCR, + .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, + .tcr_disabled = 0, + .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, + + .ttbr = DART_T8020_TTBR, + .ttbr_valid = DART_T8020_TTBR_VALID, + .ttbr_addr_off = DART_T8020_TTBR_ADDR_OFF, + .ttbr_shift = DART_T8020_TTBR_SHIFT, + .ttbr_count = 4, }; static const struct apple_dart_hw apple_dart_hw_t6000 = { + .type = DART_T6000, + .irq_handler = apple_dart_t8020_irq, + .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, .oas = 42, .fmt = APPLE_DART2, + .max_sid_count = 16, + + .enable_streams = DART_T8020_STREAMS_ENABLE, + .lock = DART_T8020_CONFIG, + .lock_bit = DART_T8020_CONFIG_LOCK, + + .error = DART_T8020_ERROR, + + .tcr = DART_T8020_TCR, + .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, + .tcr_disabled = 0, + .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, + + .ttbr = DART_T8020_TTBR, + .ttbr_valid = DART_T8020_TTBR_VALID, + .ttbr_addr_off = DART_T8020_TTBR_ADDR_OFF, + .ttbr_shift = DART_T8020_TTBR_SHIFT, + .ttbr_count = 4, +}; + +static const struct apple_dart_hw apple_dart_hw_t8110 = { + .type = DART_T8110, + .irq_handler = apple_dart_t8110_irq, + .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, + .fmt = APPLE_DART2, + .max_sid_count = 256, + + .enable_streams = DART_T8110_ENABLE_STREAMS, + .disable_streams = DART_T8110_DISABLE_STREAMS, + .lock = DART_T8110_PROTECT, + .lock_bit = DART_T8110_PROTECT_TTBR_TCR, + + .error = DART_T8110_ERROR, + + .tcr = DART_T8110_TCR, + .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, + .tcr_disabled = 0, + .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, + + .ttbr = DART_T8110_TTBR, + .ttbr_valid = DART_T8110_TTBR_VALID, + .ttbr_addr_off = DART_T8110_TTBR_ADDR_OFF, + .ttbr_shift = DART_T8110_TTBR_SHIFT, + .ttbr_count = 1, }; +#ifdef CONFIG_PM_SLEEP +static int apple_dart_suspend(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = readl_relaxed(dart->regs + DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + dart->save_ttbr[sid][idx] = + readl_relaxed(dart->regs + DART_TTBR(dart, sid, idx)); + } + + return 0; +} + +static int apple_dart_resume(struct device *dev) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned int sid, idx; + int ret; + + ret = apple_dart_hw_reset(dart); + if (ret) { + dev_err(dev, "Failed to reset DART on resume\n"); + return ret; + } + + for (sid = 0; sid < dart->num_streams; sid++) { + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + writel_relaxed(dart->save_ttbr[sid][idx], + dart->regs + DART_TTBR(dart, sid, idx)); + writel_relaxed(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); + } + + return 0; +} + +static const struct dev_pm_ops apple_dart_pm_ops = { + .suspend = apple_dart_suspend, + .resume = apple_dart_resume, +}; +#endif + static const struct of_device_id apple_dart_of_match[] = { + { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, {}, @@ -944,6 +1361,9 @@ static struct platform_driver apple_dart_driver = { .name = "apple-dart", .of_match_table = apple_dart_of_match, .suppress_bind_attrs = true, +#ifdef CONFIG_PM_SLEEP + .pm = &apple_dart_pm_ops, +#endif }, .probe = apple_dart_probe, .remove = apple_dart_remove, |