M7350/kernel/drivers/iommu/msm_iommu_mapping.c
2024-09-09 08:57:42 +00:00

642 lines
15 KiB
C

/* Copyright (c) 2011-2015, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
*/
#include <linux/dma-buf.h>
#include <linux/export.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
#include <linux/kref.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/msm_iommu_domains.h>
enum {
DI_PARTITION_NUM = 0,
DI_DOMAIN_NUM = 1,
DI_MAX,
};
#define iommu_map_domain(__m) ((__m)->domain_info[1])
#define iommu_map_partition(__m) ((__m)->domain_info[0])
/**
* struct msm_iommu_map - represents a mapping of an ion buffer to an iommu
* @iova_addr - iommu virtual address
* @node - rb node to exist in the buffer's tree of iommu mappings
* @domain_info - contains the partition number and domain number
* domain_info[1] = domain number
* domain_info[0] = partition number
* @ref - for reference counting this mapping
* @mapped_size - size of the iova space mapped
* (may not be the same as the buffer size)
* @flags - iommu domain/partition specific flags.
*
* Represents a mapping of one ion buffer to a particular iommu domain
* and address range. There may exist other mappings of this buffer in
* different domains or address ranges. All mappings will have the same
* cacheability and security.
*/
struct msm_iommu_map {
unsigned long iova_addr;
struct rb_node node;
union {
int domain_info[DI_MAX];
uint64_t key;
};
struct msm_iommu_meta *meta;
struct kref ref;
int mapped_size;
unsigned long flags;
};
struct msm_iommu_meta {
struct rb_node node;
struct rb_root iommu_maps;
struct kref ref;
struct sg_table *table;
unsigned long size;
struct mutex lock;
struct dma_buf *dbuf;
};
static struct rb_root iommu_root;
DEFINE_MUTEX(msm_iommu_map_mutex);
static void msm_iommu_meta_add(struct msm_iommu_meta *meta)
{
struct rb_root *root = &iommu_root;
struct rb_node **p = &root->rb_node;
struct rb_node *parent = NULL;
struct msm_iommu_meta *entry;
while (*p) {
parent = *p;
entry = rb_entry(parent, struct msm_iommu_meta, node);
if (meta->table < entry->table) {
p = &(*p)->rb_left;
} else if (meta->table > entry->table) {
p = &(*p)->rb_right;
} else {
pr_err("%s: dma_buf %p already exists\n", __func__,
entry->dbuf);
BUG();
}
}
rb_link_node(&meta->node, parent, p);
rb_insert_color(&meta->node, root);
}
static struct msm_iommu_meta *msm_iommu_meta_lookup(struct sg_table *table)
{
struct rb_root *root = &iommu_root;
struct rb_node **p = &root->rb_node;
struct rb_node *parent = NULL;
struct msm_iommu_meta *entry = NULL;
while (*p) {
parent = *p;
entry = rb_entry(parent, struct msm_iommu_meta, node);
if (table < entry->table)
p = &(*p)->rb_left;
else if (table > entry->table)
p = &(*p)->rb_right;
else
return entry;
}
return NULL;
}
static void msm_iommu_add(struct msm_iommu_meta *meta,
struct msm_iommu_map *iommu)
{
struct rb_node **p = &meta->iommu_maps.rb_node;
struct rb_node *parent = NULL;
struct msm_iommu_map *entry;
while (*p) {
parent = *p;
entry = rb_entry(parent, struct msm_iommu_map, node);
if (iommu->key < entry->key) {
p = &(*p)->rb_left;
} else if (iommu->key > entry->key) {
p = &(*p)->rb_right;
} else {
pr_err("%s: dma_buf %p already has mapping for domain %d and partition %d\n",
__func__,
meta->dbuf,
iommu_map_domain(iommu),
iommu_map_partition(iommu));
BUG();
}
}
rb_link_node(&iommu->node, parent, p);
rb_insert_color(&iommu->node, &meta->iommu_maps);
}
static struct msm_iommu_map *msm_iommu_lookup(
struct msm_iommu_meta *meta,
unsigned int domain_no,
unsigned int partition_no)
{
struct rb_node **p = &meta->iommu_maps.rb_node;
struct rb_node *parent = NULL;
struct msm_iommu_map *entry;
uint64_t key = domain_no;
key = key << 32 | partition_no;
while (*p) {
parent = *p;
entry = rb_entry(parent, struct msm_iommu_map, node);
if (key < entry->key)
p = &(*p)->rb_left;
else if (key > entry->key)
p = &(*p)->rb_right;
else
return entry;
}
return NULL;
}
static int msm_iommu_map_iommu(struct msm_iommu_meta *meta,
struct msm_iommu_map *data,
unsigned int domain_num,
unsigned int partition_num,
unsigned long align,
unsigned long iova_length,
unsigned long flags)
{
struct iommu_domain *domain;
int ret = 0;
unsigned long extra, size;
struct sg_table *table;
int prot = IOMMU_WRITE | IOMMU_READ;
size_t map_ret;
size = meta->size;
data->mapped_size = iova_length;
extra = iova_length - size;
table = meta->table;
/* Use the biggest alignment to allow bigger IOMMU mappings.
* Use the first entry since the first entry will always be the
* biggest entry. To take advantage of bigger mapping sizes both the
* VA and PA addresses have to be aligned to the biggest size.
*/
if (table->sgl->length > align)
align = table->sgl->length;
ret = msm_allocate_iova_address(domain_num, partition_num,
data->mapped_size, align,
&data->iova_addr);
if (ret)
goto out;
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
ret = -ENOMEM;
goto out1;
}
map_ret = iommu_map_sg(domain, data->iova_addr,
table->sgl,
table->nents, prot);
if (map_ret != size) {
pr_err("%s: could not map %lx in domain %p\n",
__func__, data->iova_addr, domain);
ret = -EINVAL;
goto out1;
} else {
ret = 0;
}
if (extra) {
unsigned long extra_iova_addr = data->iova_addr + size;
unsigned long phys_addr = sg_phys(table->sgl);
ret = msm_iommu_map_extra(domain, extra_iova_addr, phys_addr,
extra, SZ_4K, prot);
if (ret)
goto out2;
}
return ret;
out2:
iommu_unmap_range(domain, data->iova_addr, size);
out1:
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
size);
out:
return ret;
}
static void msm_iommu_heap_unmap_iommu(struct msm_iommu_map *data)
{
unsigned int domain_num;
unsigned int partition_num;
struct iommu_domain *domain;
int ret;
BUG_ON(!msm_use_iommu());
domain_num = iommu_map_domain(data);
partition_num = iommu_map_partition(data);
domain = msm_get_iommu_domain(domain_num);
if (!domain) {
WARN(1, "Could not get domain %d. Corruption?\n", domain_num);
return;
}
ret = iommu_unmap_range(domain, data->iova_addr, data->mapped_size);
WARN_ON(ret < 0);
msm_free_iova_address(data->iova_addr, domain_num, partition_num,
data->mapped_size);
}
static struct msm_iommu_map *__msm_iommu_map(struct msm_iommu_meta *meta,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, unsigned long flags,
ion_phys_addr_t *iova)
{
struct msm_iommu_map *data;
int ret;
data = kmalloc(sizeof(*data), GFP_ATOMIC);
if (!data)
return ERR_PTR(-ENOMEM);
iommu_map_domain(data) = domain_num;
iommu_map_partition(data) = partition_num;
ret = msm_iommu_map_iommu(meta, data,
domain_num,
partition_num,
align,
iova_length,
flags);
if (ret)
goto out;
kref_init(&data->ref);
*iova = data->iova_addr;
data->meta = meta;
msm_iommu_add(meta, data);
return data;
out:
kfree(data);
return ERR_PTR(ret);
}
static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf,
struct sg_table *table,
unsigned long size)
{
struct msm_iommu_meta *meta;
meta = kzalloc(sizeof(*meta), GFP_KERNEL);
if (!meta)
return ERR_PTR(-ENOMEM);
meta->table = table;
meta->size = size;
/*
* The caller is expected to have taken a reference to this dma_buf
* before calling this function
*/
meta->dbuf = dma_buf;
kref_init(&meta->ref);
mutex_init(&meta->lock);
msm_iommu_meta_add(meta);
return meta;
}
static void msm_iommu_meta_destroy(struct kref *kref)
{
struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta,
ref);
rb_erase(&meta->node, &iommu_root);
dma_buf_put(meta->dbuf);
kfree(meta);
}
static void msm_iommu_meta_put(struct msm_iommu_meta *meta)
{
/*
* Need to lock here to prevent race against map/unmap
*/
mutex_lock(&msm_iommu_map_mutex);
kref_put(&meta->ref, msm_iommu_meta_destroy);
mutex_unlock(&msm_iommu_map_mutex);
}
static int __msm_map_iommu_common(
struct dma_buf *dma_buf, struct sg_table *table,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, ion_phys_addr_t *iova,
unsigned long *buffer_size,
unsigned long flags, unsigned long iommu_flags)
{
struct msm_iommu_map *iommu_map;
struct msm_iommu_meta *iommu_meta = NULL;
struct scatterlist *sg;
unsigned long size = 0;
int ret = 0;
int i;
for_each_sg(table->sgl, sg, table->nents, i)
size += sg->length;
if (!msm_use_iommu()) {
unsigned long pa = sg_dma_address(table->sgl);
if (pa == 0)
pa = sg_phys(table->sgl);
*iova = pa;
*buffer_size = size;
}
/*
* If clients don't want a custom iova length, just use whatever
* the buffer size is
*/
if (!iova_length)
iova_length = size;
if (size > iova_length) {
pr_debug("%s: iova length %lx is not at least buffer size %lx\n",
__func__, iova_length, size);
ret = -EINVAL;
goto out;
}
if (size & ~PAGE_MASK) {
pr_debug("%s: buffer size %lx is not aligned to %lx", __func__,
size, PAGE_SIZE);
ret = -EINVAL;
goto out;
}
if (iova_length & ~PAGE_MASK) {
pr_debug("%s: iova_length %lx is not aligned to %lx", __func__,
iova_length, PAGE_SIZE);
ret = -EINVAL;
goto out;
}
mutex_lock(&msm_iommu_map_mutex);
iommu_meta = msm_iommu_meta_lookup(table);
if (!iommu_meta) {
iommu_meta = msm_iommu_meta_create(dma_buf, table, size);
if (IS_ERR(iommu_meta)) {
mutex_unlock(&msm_iommu_map_mutex);
ret = PTR_ERR(iommu_meta);
goto out;
}
} else {
/*
* Drop the dma_buf reference here. We took the reference
* during meta creation so we need to drop it if we are
* just taking a reference to the meta itself.
*/
dma_buf_put(dma_buf);
kref_get(&iommu_meta->ref);
}
BUG_ON(iommu_meta->size != size);
mutex_unlock(&msm_iommu_map_mutex);
mutex_lock(&iommu_meta->lock);
iommu_map = msm_iommu_lookup(iommu_meta, domain_num, partition_num);
if (!iommu_map) {
iommu_map = __msm_iommu_map(iommu_meta, domain_num,
partition_num, align, iova_length,
flags, iova);
if (!IS_ERR_OR_NULL(iommu_map)) {
iommu_map->flags = iommu_flags;
ret = 0;
} else {
ret = PTR_ERR(iommu_map);
goto out_unlock;
}
} else {
if (iommu_map->flags != iommu_flags) {
pr_err("%s: dma_buf %p is already mapped with iommu flags %lx, trying to map with flags %lx\n",
__func__, dma_buf,
iommu_map->flags, iommu_flags);
ret = -EINVAL;
goto out_unlock;
} else if (iommu_map->mapped_size != iova_length) {
pr_err("%s: dma_buf %p is already mapped with length %x, trying to map with length %lx\n",
__func__, dma_buf, iommu_map->mapped_size,
iova_length);
ret = -EINVAL;
goto out_unlock;
} else {
kref_get(&iommu_map->ref);
*iova = iommu_map->iova_addr;
}
}
mutex_unlock(&iommu_meta->lock);
*buffer_size = size;
return ret;
out_unlock:
mutex_unlock(&iommu_meta->lock);
out:
if (!IS_ERR(iommu_meta))
msm_iommu_meta_put(iommu_meta);
return ret;
}
int msm_map_dma_buf(struct dma_buf *dma_buf, struct sg_table *table,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, ion_phys_addr_t *iova,
unsigned long *buffer_size,
unsigned long flags, unsigned long iommu_flags)
{
int ret;
if (IS_ERR_OR_NULL(dma_buf)) {
pr_err("%s: dma_buf pointer is invalid\n", __func__);
return -EINVAL;
}
if (IS_ERR_OR_NULL(table)) {
pr_err("%s: table pointer is invalid\n", __func__);
return -EINVAL;
}
get_dma_buf(dma_buf);
ret = __msm_map_iommu_common(dma_buf, table, domain_num,
partition_num, align, iova_length, iova,
buffer_size, flags, iommu_flags);
if (ret)
dma_buf_put(dma_buf);
return ret;
}
int ion_map_iommu(struct ion_client *client, struct ion_handle *handle,
int domain_num, int partition_num, unsigned long align,
unsigned long iova_length, ion_phys_addr_t *iova,
unsigned long *buffer_size,
unsigned long flags, unsigned long iommu_flags)
{
struct sg_table *table;
struct dma_buf *dma_buf;
int ret = 0;
if (IS_ERR_OR_NULL(client)) {
pr_err("%s: client pointer is invalid\n", __func__);
return -EINVAL;
}
if (IS_ERR_OR_NULL(handle)) {
pr_err("%s: handle pointer is invalid\n", __func__);
return -EINVAL;
}
table = ion_sg_table(client, handle);
if (IS_ERR(table))
return PTR_ERR(table);
dma_buf = ion_share_dma_buf(client, handle);
if (IS_ERR(dma_buf))
return PTR_ERR(dma_buf);
ret = __msm_map_iommu_common(dma_buf, table, domain_num,
partition_num, align, iova_length, iova,
buffer_size, flags, iommu_flags);
if (ret)
dma_buf_put(dma_buf);
return ret;
}
EXPORT_SYMBOL(ion_map_iommu);
static void msm_iommu_map_release(struct kref *kref)
{
struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map,
ref);
struct msm_iommu_meta *meta = map->meta;
rb_erase(&map->node, &meta->iommu_maps);
msm_iommu_heap_unmap_iommu(map);
kfree(map);
}
static void __msm_unmap_iommu_common(struct sg_table *table, int domain_num,
int partition_num)
{
struct msm_iommu_map *iommu_map;
struct msm_iommu_meta *meta;
mutex_lock(&msm_iommu_map_mutex);
meta = msm_iommu_meta_lookup(table);
if (!meta) {
WARN(1, "%s: (%d,%d) was never mapped for %p\n", __func__,
domain_num, partition_num, table);
mutex_unlock(&msm_iommu_map_mutex);
goto out;
}
mutex_unlock(&msm_iommu_map_mutex);
mutex_lock(&meta->lock);
iommu_map = msm_iommu_lookup(meta, domain_num, partition_num);
if (!iommu_map) {
WARN(1, "%s: (%d,%d) was never mapped for %p\n", __func__,
domain_num, partition_num, table);
mutex_unlock(&meta->lock);
goto out;
}
kref_put(&iommu_map->ref, msm_iommu_map_release);
mutex_unlock(&meta->lock);
msm_iommu_meta_put(meta);
out:
return;
}
void msm_unmap_dma_buf(struct sg_table *table, int domain_num,
int partition_num)
{
return __msm_unmap_iommu_common(table, domain_num, partition_num);
}
void ion_unmap_iommu(struct ion_client *client, struct ion_handle *handle,
int domain_num, int partition_num)
{
struct sg_table *table;
if (IS_ERR_OR_NULL(client)) {
pr_err("%s: client pointer is invalid\n", __func__);
return;
}
if (IS_ERR_OR_NULL(handle)) {
pr_err("%s: handle pointer is invalid\n", __func__);
return;
}
table = ion_sg_table(client, handle);
__msm_unmap_iommu_common(table, domain_num, partition_num);
}
EXPORT_SYMBOL(ion_unmap_iommu);