diff --git a/Makefile b/Makefile index edfc2fd..a3586b7 100644 --- a/Makefile +++ b/Makefile @@ -552,7 +552,7 @@ export mod_strip_cmd ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ acrypto/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/acrypto/Kconfig b/acrypto/Kconfig new file mode 100644 index 0000000..0023cd0 --- /dev/null +++ b/acrypto/Kconfig @@ -0,0 +1,36 @@ +menu "Asynchronous crypto layer" + +config ACRYPTO + tristate "Asynchronous crypto layer" + select CONNECTOR + --- help --- + It supports: + - multiple asynchronous crypto device queues + - crypto session routing + - crypto session binding + - modular load balancing + - crypto session batching genetically implemented by design + - crypto session priority + - different kinds of crypto operation(RNG, asymmetrical crypto, HMAC and any other + +config ASYNC_PROVIDER + tristate "Asynchronous crypto provider (AES CBC)" + depends on ACRYPTO && (CRYPTO_AES || CRYPTO_AES_586 || CRYPTO_AES_X86_64) + --- help --- + Asynchronous crypto provider based on synchronous crypto layer. + It supports AES CBC crypto mode (may be changed by source edition). + +config CONSUMER + tristate "Test asynchronous crypto consumer" + depends on ACRYPTO + --- help --- + Test asynchronous crypto consumer. + +config ASYNC2OCF_BRIDGE + tristate "Async to OCF bridge driver" + depends on ACRYPTO && (CRYPTO_AES || CRYPTO_AES_586 || CRYPTO_AES_X86_64) + --- help --- + Acrypto driver which resorts to asynchronous OCF layer for + encryption functionality. + +endmenu diff --git a/acrypto/Makefile b/acrypto/Makefile new file mode 100644 index 0000000..e819c48 --- /dev/null +++ b/acrypto/Makefile @@ -0,0 +1,15 @@ +obj-$(CONFIG_ACRYPTO) += acrypto.o +obj-$(CONFIG_ASYNC_PROVIDER) += async_provider.o +obj-$(CONFIG_CONSUMER) += consumer.o +obj-$(CONFIG_ASYNC2OCF_BRIDGE) += ocf_acrypto.o + +acrypto-y += acrypto_main.o +acrypto-y += acrypto_lb.o +acrypto-y += acrypto_dev.o +acrypto-y += acrypto_conn.o +acrypto-y += acrypto_stat.o +acrypto-y += acrypto_user_direct.o +acrypto-y += acrypto_user_ioctl.o +acrypto-y += acrypto_user.o +acrypto-y += simple_lb.o +acrypto-y += acrypto_queue.o diff --git a/acrypto/acrypto_conn.c b/acrypto/acrypto_conn.c new file mode 100644 index 0000000..90474fd --- /dev/null +++ b/acrypto/acrypto_conn.c @@ -0,0 +1,135 @@ +/* + * acrypto_conn.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "acrypto_lb.h" + +#include + +#include "acrypto_conn.h" +#include "acrypto_user_ioctl.h" +#include "acrypto_user_direct.h" + +struct cb_id acrypto_conn_id = { 0xdead, 0x0000 }; +static char acrypto_conn_name[] = "crconn"; + +static void acrypto_conn_callback(void *data) +{ + struct cn_msg *msg, *reply; + struct acrypto_conn_data *d, *cmd; + struct acrypto_device *dev; + + msg = (struct cn_msg *)data; + d = (struct acrypto_conn_data *)msg->data; + + if (msg->len < sizeof(*d)) { + dprintk(KERN_ERR "Wrong message to acrypto connector: msg->len=%u < %zu.\n", + msg->len, sizeof(*d)); + return; + } + + if (msg->len != sizeof(*d) + d->len) { + dprintk(KERN_ERR "Wrong message to acrypto connector: msg->len=%u != %zu.\n", + msg->len, sizeof(*d) + d->len); + return; + } + + dev = acrypto_device_get_name(d->name); + if (!dev) { + dprintk("Crypto device %s was not found.\n", d->name); + return; + } + + switch (d->cmd) { + case ACRYPTO_GET_STAT: + reply = kmalloc(sizeof(*msg) + sizeof(*cmd) + sizeof(struct acrypto_device_stat), GFP_ATOMIC); + if (reply) { + struct acrypto_device_stat *ptr; + + memcpy(reply, msg, sizeof(*reply)); + reply->len = sizeof(*cmd) + sizeof(*ptr); + + /* + * See protocol description in connector.c + */ + reply->ack++; + + cmd = (struct acrypto_conn_data *)(reply + 1); + memcpy(cmd, d, sizeof(*cmd)); + cmd->len = sizeof(*ptr); + + ptr = (struct acrypto_device_stat *)(cmd + 1); + memcpy(ptr, &dev->stat, sizeof(*ptr)); + + cn_netlink_send(reply, 0, GFP_ATOMIC); + + kfree(reply); + } else + dprintk(KERN_ERR "Failed to allocate %zu bytes in reply to comamnd 0x%x.\n", + sizeof(*msg) + sizeof(*cmd), d->cmd); + break; + case ACRYPTO_REQUEST: +#if 1 + { + struct acrypto_user_direct *usr; + + usr = (struct acrypto_user_direct *)(d->data); + + acrypto_user_direct_add_request(msg->seq, msg->ack, usr); + } +#endif + break; + default: + dprintk(KERN_ERR "Wrong operation 0x%04x for acrypto connector.\n", + d->cmd); + return; + } + + acrypto_device_put(dev); +} + +int acrypto_conn_init(void) +{ + int err; + + err = cn_add_callback(&acrypto_conn_id, acrypto_conn_name, acrypto_conn_callback); + if (err) + return err; + + dprintk("Crypto connector callback is registered.\n"); + + return 0; +} + +void acrypto_conn_fini(void) +{ + cn_del_callback(&acrypto_conn_id); + dprintk("Crypto connector callback is unregistered.\n"); +} diff --git a/acrypto/acrypto_conn.h b/acrypto/acrypto_conn.h new file mode 100644 index 0000000..91ed7da --- /dev/null +++ b/acrypto/acrypto_conn.h @@ -0,0 +1,44 @@ +/* + * acrypto_conn.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_CONN_H +#define __ACRYPTO_CONN_H + +#include + +#define ACRYPTO_REQUEST 1 +#define ACRYPTO_GET_STAT 2 + +struct acrypto_conn_data +{ + char name[SCACHE_NAMELEN]; + __u16 cmd; + __u16 len; + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +int acrypto_conn_init(void); +void acrypto_conn_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __ACRYPTO_CONN_H */ diff --git a/acrypto/acrypto_dev.c b/acrypto/acrypto_dev.c new file mode 100644 index 0000000..b8a2ffb --- /dev/null +++ b/acrypto/acrypto_dev.c @@ -0,0 +1,507 @@ +/* + * acrypto_dev.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +LIST_HEAD(cdev_list); +spinlock_t cdev_lock = SPIN_LOCK_UNLOCKED; +static u32 cdev_ids; +extern unsigned int max_ca_drivers; + +static int acrypto_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int acrypto_probe(struct device *dev) +{ + return -ENODEV; +} + +static int acrypto_remove(struct device *dev) +{ + return 0; +} + +static void acrypto_release(struct device *dev) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, device); + + complete(&d->dev_released); +} + +static void acrypto_class_release(struct class *class) +{ +} + +static void acrypto_class_release_device(struct class_device *class_dev) +{ +} + +struct class acrypto_class = { + .name = "acrypto", + .class_release = acrypto_class_release, + .release = acrypto_class_release_device +}; + +struct bus_type acrypto_bus_type = { + .name = "acrypto", + .match = acrypto_match +}; + +struct device_driver acrypto_driver = { + .name = "acrypto_driver", + .bus = &acrypto_bus_type, + .probe = acrypto_probe, + .remove = acrypto_remove, +}; + +struct device acrypto_dev = { + .parent = NULL, + .bus = &acrypto_bus_type, + .bus_id = "Asynchronous acrypto", + .driver = &acrypto_driver, + .release = &acrypto_release +}; + +static ssize_t sessions_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%d\n", atomic_read(&d->refcnt)); +} +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%s\n", d->name); +} +static ssize_t devices_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(d, &cdev_list, cdev_entry) { + off += sprintf(buf + off, "%s ", d->name); + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!off) + off = sprintf(buf, "No devices registered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} + +static ssize_t queue_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + unsigned long flags; + struct acrypto_queue *q; + int size = PAGE_SIZE, ret; + + spin_lock_irqsave(&d->queue.queue_lock, flags); + list_for_each_entry(q, &d->queue.queue_list, queue_entry) { + ret = snprintf(buf, size, "%4u - %u\n", q->priority, q->qlen); + size -= ret; + buf += ret; + } + spin_unlock_irqrestore(&d->queue.queue_lock, flags); + return PAGE_SIZE - size; +} +static ssize_t kmem_failed_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.kmem_failed); +} +static ssize_t sstarted_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sstarted); +} +static ssize_t sfinished_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sfinished); +} +static ssize_t scompleted_show(struct class_device *dev, char *buf) +{ + struct acrypto_device *d = container_of(dev, struct acrypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.scompleted); +} + +static CLASS_DEVICE_ATTR(sessions, 0444, sessions_show, NULL); +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(devices, 0444, devices_show, NULL); +static CLASS_DEVICE_ATTR(scompleted, 0444, scompleted_show, NULL); +static CLASS_DEVICE_ATTR(sstarted, 0444, sstarted_show, NULL); +static CLASS_DEVICE_ATTR(sfinished, 0444, sfinished_show, NULL); +static CLASS_DEVICE_ATTR(kmem_failed, 0444, kmem_failed_show, NULL); +static CLASS_DEVICE_ATTR(queue, 0444, queue_show, NULL); + +static void create_device_attributes(struct acrypto_device *dev) +{ + class_device_create_file(&dev->class_device, &class_device_attr_sessions); + class_device_create_file(&dev->class_device, &class_device_attr_name); + class_device_create_file(&dev->class_device, &class_device_attr_scompleted); + class_device_create_file(&dev->class_device, &class_device_attr_sstarted); + class_device_create_file(&dev->class_device, &class_device_attr_sfinished); + class_device_create_file(&dev->class_device, &class_device_attr_kmem_failed); + class_device_create_file(&dev->class_device, &class_device_attr_queue); +} + +static void remove_device_attributes(struct acrypto_device *dev) +{ + class_device_remove_file(&dev->class_device, &class_device_attr_sessions); + class_device_remove_file(&dev->class_device, &class_device_attr_name); + class_device_remove_file(&dev->class_device, &class_device_attr_scompleted); + class_device_remove_file(&dev->class_device, &class_device_attr_sstarted); + class_device_remove_file(&dev->class_device, &class_device_attr_sfinished); + class_device_remove_file(&dev->class_device, &class_device_attr_kmem_failed); + class_device_remove_file(&dev->class_device, &class_device_attr_queue); +} + +int __match_initializer(struct acrypto_capability *cap, struct acrypto_session_initializer *ci, struct acrypto_device *dev) +{ + unsigned int ref = atomic_read(&dev->refcnt) + 1; + int ret = 0; + + if (cap->operation == ci->operation && cap->type == ci->type && + cap->mode == (ci->mode & 0x1fff) && + cap->qlen >= ref) { + ret = 1; + dprintk("Match: %04x.%04x.%04x vs. %04x.%04x.%04x, max_len=%u, ref=%u.\n", + cap->operation, cap->type, cap->mode, + ci->operation, ci->type, ci->mode, + cap->qlen, ref); + } + + return ret; +} + +int match_initializer(struct acrypto_device *dev, struct acrypto_session_initializer *ci) +{ + int i; + + for (i = 0; i < dev->cap_number; ++i) { + struct acrypto_capability *cap = &dev->cap[i]; + + if (__match_initializer(cap, ci, dev)) + return 1; + } + + return 0; +} + +void acrypto_device_get(struct acrypto_device *dev) +{ + atomic_inc(&dev->refcnt); +} + +struct acrypto_device *acrypto_device_get_name(char *name) +{ + struct acrypto_device *dev; + int found = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (!strcmp(dev->name, name)) { + found = 1; + acrypto_device_get(dev); + break; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!found) + return NULL; + + return dev; +} + +void acrypto_device_put(struct acrypto_device *dev) +{ + atomic_dec(&dev->refcnt); +} + +static int avg_cap_qlen(struct acrypto_device *dev) +{ + int i, max = 0; + + if (!dev->cap_number) + return 0; + + for (i=0; icap_number; ++i) { + max += dev->cap[i].qlen; + } + + return (max / dev->cap_number); +} + +static int is_ca_index_taken(int index) +{ + struct acrypto_device *dev; + unsigned long flags; + int result = 0; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if( dev->ca_index == index ) { + result = 1; + break; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + return result; +} + +int __acrypto_device_add(struct acrypto_device *dev) +{ + int err = -ENOMEM, avg; + + memset(&dev->stat, 0, sizeof(dev->stat)); + spin_lock_init(&dev->stat_lock); + spin_lock_init(&dev->lock); + atomic_set(&dev->refcnt, 0); + atomic_set(&dev->sid, 0); + init_completion(&dev->dev_released); + memcpy(&dev->device, &acrypto_dev, sizeof(struct device)); + dev->driver = &acrypto_driver; + INIT_LIST_HEAD(&dev->queue.queue_list); + spin_lock_init(&dev->queue.queue_lock); + + avg = avg_cap_qlen(dev); + if (!avg) + return -EINVAL; + + dev->ca_index = 0; /* ca_index is used only by context-aware drivers */ + if( dev->flags & ACRYPTO_DEV_CTX_INIT) { + while ( is_ca_index_taken( dev->ca_index ) && + dev->ca_index < max_ca_drivers) + dev->ca_index++; + + if( dev->ca_index >= max_ca_drivers ) { + printk(KERN_ERR "Too many context-aware acrypto drivers (%d). " + "Failed to load driver %s.\n" + "Please reload acrypto with larger value of max_ca_drivers.\n", + max_ca_drivers, dev->name ); + return -EINVAL; + } + } + + dev->session_cache = kmem_cache_create(dev->name, sizeof(struct acrypto_session), + 0, 0, NULL, NULL); + if (!dev->session_cache) { + dprintk(KERN_ERR "Failed to create session cache for device %s.\n", dev->name); + return err; + } + + dev->session_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->session_cache); + if (!dev->session_pool) { + dprintk(KERN_ERR "Failed to create session memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_cache_destroy; + } + + snprintf(dev->route_cache_name, sizeof(dev->route_cache_name), "%s-route", dev->name); + dev->route_cache = kmem_cache_create(dev->route_cache_name, sizeof(struct acrypto_route), 0, 0, NULL, NULL); + if (!dev->route_cache) { + dprintk(KERN_ERR "Failed to create route cache for device %s.\n", dev->name); + goto err_out_mempool_destroy; + } + + avg = (avg)?avg*2:1024; + dev->route_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->route_cache); + if (!dev->route_pool) { + dprintk(KERN_ERR "Failed to create route memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_route_cache_destroy; + } + + snprintf(dev->queue_cache_name, sizeof(dev->queue_cache_name), "%s-queue", dev->name); + dev->queue_cache = kmem_cache_create(dev->queue_cache_name, sizeof(struct acrypto_queue), 0, 0, NULL, NULL); + if (!dev->queue_cache) { + dprintk(KERN_ERR "Failed to create queue cache for device %s.\n", dev->name); + goto err_out_route_mempool_destroy; + } + + dev->queue_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->queue_cache); + if (!dev->queue_pool) { + dprintk(KERN_ERR "Failed to create queue memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_queue_cache_destroy; + } + + snprintf(dev->device.bus_id, sizeof(dev->device.bus_id), "%s", dev->name); + err = device_register(&dev->device); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto device %s: err=%d.\n", + dev->name, err); + goto err_out_queue_mempool_destroy; + } + + snprintf(dev->class_device.class_id, sizeof(dev->class_device.class_id), "%s", dev->name); + dev->class_device.dev = &dev->device; + dev->class_device.class = &acrypto_class; + + err = class_device_register(&dev->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto class device %s: err=%d.\n", + dev->name, err); + goto err_out_device_unregister; + } + + create_device_attributes(dev); + + return 0; + +err_out_device_unregister: + device_unregister(&dev->device); +err_out_queue_mempool_destroy: + mempool_destroy(dev->queue_pool); +err_out_queue_cache_destroy: + kmem_cache_destroy(dev->queue_cache); +err_out_route_mempool_destroy: + mempool_destroy(dev->route_pool); +err_out_route_cache_destroy: + kmem_cache_destroy(dev->route_cache); +err_out_mempool_destroy: + mempool_destroy(dev->session_pool); +err_out_cache_destroy: + kmem_cache_destroy(dev->session_cache); + + return err; +} + +void __acrypto_device_remove(struct acrypto_device *dev) +{ + remove_device_attributes(dev); + class_device_unregister(&dev->class_device); + device_unregister(&dev->device); + + acrypto_destroy_queues(dev); + + mempool_destroy(dev->route_pool); + kmem_cache_destroy(dev->route_cache); + mempool_destroy(dev->session_pool); + kmem_cache_destroy(dev->session_cache); + mempool_destroy(dev->queue_pool); + kmem_cache_destroy(dev->queue_cache); +} + +int acrypto_device_add(struct acrypto_device *dev) +{ + int err; + unsigned long flags; + + err = __acrypto_device_add(dev); + if (err) + return err; + + spin_lock_irqsave(&cdev_lock, flags); + list_add(&dev->cdev_entry, &cdev_list); + dev->id = ++cdev_ids; + spin_unlock_irqrestore(&cdev_lock, flags); + + printk("Crypto device %s was registered with ID=%x.\n", + dev->name, dev->id); + + return 0; +} + +void acrypto_device_remove(struct acrypto_device *dev) +{ + struct acrypto_device *__dev, *n; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry_safe(__dev, n, &cdev_list, cdev_entry) { + if (compare_device(__dev, dev)) { + list_del_init(&__dev->cdev_entry); + spin_unlock_irqrestore(&cdev_lock, flags); + + + /* + * In test cases or when acrypto device driver is not written correctly, + * it's ->data_ready() method will not be called anymore + * after device is removed from acrypto device list. + * + * For such cases we either should provide ->flush() call + * or properly write ->data_ready() method. + */ + + while (atomic_read(&__dev->refcnt)) { + + dprintk("Waiting for %s to become free: refcnt=%d.\n", + __dev->name, atomic_read(&dev->refcnt)); + + /* + * Hack zone: you need to write good ->data_ready() + * and acrypto device driver itself. + * + * Driver shoud not buzz if it has pending sessions + * in it's queue and it was removed from global device list. + * + * Although I can workaround it here, for example by + * flushing the whole queue and drop all pending sessions. + */ + + __dev->data_ready(__dev); + ssleep(1); + } + + __acrypto_device_remove(dev); + + dprintk(KERN_ERR "Crypto device %s was unregistered.\n", + dev->name); + return; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + dprintk(KERN_ERR "Crypto device %s was not registered.\n", dev->name); +} + +EXPORT_SYMBOL_GPL(acrypto_device_add); +EXPORT_SYMBOL_GPL(acrypto_device_remove); +EXPORT_SYMBOL_GPL(acrypto_device_get); +EXPORT_SYMBOL_GPL(acrypto_device_get_name); +EXPORT_SYMBOL_GPL(acrypto_device_put); +EXPORT_SYMBOL_GPL(match_initializer); diff --git a/acrypto/acrypto_lb.c b/acrypto/acrypto_lb.c new file mode 100644 index 0000000..6204d44 --- /dev/null +++ b/acrypto/acrypto_lb.c @@ -0,0 +1,504 @@ +/* + * acrypto_lb.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "acrypto_lb.h" +#include "acrypto_stat.h" +#include "acrypto_route.h" + +static LIST_HEAD(acrypto_lb_list); +static spinlock_t acrypto_lb_lock = SPIN_LOCK_UNLOCKED; +static int lb_num = 0; +static struct acrypto_lb *current_lb, *default_lb; +static struct workqueue_struct *acrypto_lb_queue; + +extern struct list_head cdev_list; +extern spinlock_t cdev_lock; + +extern struct acrypto_lb simple_lb; + +static int lb_is_current(struct acrypto_lb *l) +{ + return (l == current_lb); +} + +static int lb_is_default(struct acrypto_lb *l) +{ + return (l == default_lb); +} + +#if 0 +static void lb_set_current(struct acrypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&acrypto_lb_lock, flags); + current_lb = l; + spin_unlock_irqrestore(&acrypto_lb_lock, flags); +} + +static void lb_set_default(struct acrypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&acrypto_lb_lock, flags); + default_lb = l; + spin_unlock_irqrestore(&acrypto_lb_lock, flags); +} +#endif +static int acrypto_lb_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int acrypto_lb_probe(struct device *dev) +{ + return -ENODEV; +} + +static int acrypto_lb_remove(struct device *dev) +{ + return 0; +} + +static void acrypto_lb_release(struct device *dev) +{ + struct acrypto_lb *d = container_of(dev, struct acrypto_lb, device); + + complete(&d->dev_released); +} + +static void acrypto_lb_class_release(struct class *class) +{ +} + +static void acrypto_lb_class_release_device(struct class_device *class_dev) +{ +} + +struct class acrypto_lb_class = { + .name = "acrypto_lb", + .class_release = acrypto_lb_class_release, + .release = acrypto_lb_class_release_device +}; + +struct bus_type acrypto_lb_bus_type = { + .name = "acrypto_lb", + .match = acrypto_lb_match +}; + +struct device_driver acrypto_lb_driver = { + .name = "acrypto_lb_driver", + .bus = &acrypto_lb_bus_type, + .probe = acrypto_lb_probe, + .remove = acrypto_lb_remove, +}; + +struct device acrypto_lb_dev = { + .parent = NULL, + .bus = &acrypto_lb_bus_type, + .bus_id = "acrypto", + .driver = &acrypto_lb_driver, + .release = &acrypto_lb_release +}; + +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct acrypto_lb *lb = container_of(dev, struct acrypto_lb, class_device); + + return sprintf(buf, "%s\n", lb->name); +} + +static ssize_t current_show(struct class_device *dev, char *buf) +{ + struct acrypto_lb *lb; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&acrypto_lb_lock, flags); + + list_for_each_entry(lb, &acrypto_lb_list, lb_entry) { + if (lb_is_current(lb)) + off += sprintf(buf + off, "["); + if (lb_is_default(lb)) + off += sprintf(buf + off, "("); + off += sprintf(buf + off, "%s", lb->name); + if (lb_is_default(lb)) + off += sprintf(buf + off, ")"); + if (lb_is_current(lb)) + off += sprintf(buf + off, "]"); + } + + spin_unlock_irqrestore(&acrypto_lb_lock, flags); + + if (!off) + off = sprintf(buf, "No load balancers regitered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} +static ssize_t current_store(struct class_device *dev, const char *buf, size_t count) +{ + struct acrypto_lb *lb; + unsigned long flags; + + spin_lock_irqsave(&acrypto_lb_lock, flags); + + list_for_each_entry(lb, &acrypto_lb_list, lb_entry) { + if (count == strlen(lb->name) && !strcmp(buf, lb->name)) { + current_lb = lb; + default_lb = lb; + + dprintk("Load balancer %s is set as current and default.\n", + lb->name); + + break; + } + } + spin_unlock_irqrestore(&acrypto_lb_lock, flags); + + return count; +} + +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(lbs, 0644, current_show, current_store); + +static void create_device_attributes(struct acrypto_lb *lb) +{ + class_device_create_file(&lb->class_device, &class_device_attr_name); +} + +static void remove_device_attributes(struct acrypto_lb *lb) +{ + class_device_remove_file(&lb->class_device, &class_device_attr_name); +} + +static int compare_lb(struct acrypto_lb *l1, struct acrypto_lb *l2) +{ + if (!strncmp(l1->name, l2->name, sizeof(l1->name))) + return 1; + + return 0; +} + +struct acrypto_device *acrypto_lb_find_device(struct acrypto_session_initializer *ci, struct acrypto_data *data) +{ + struct acrypto_device *dev; + unsigned long flags; + + if (!current_lb) + return NULL; + + if (sci_bound(ci)) { + int found = 0; + + spin_lock_irqsave(&cdev_lock, flags); + + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->id == ci->bdev) { + found = 1; + break; + } + } + + spin_unlock_irqrestore(&cdev_lock, flags); + + return (found) ? dev : NULL; + } + + spin_lock_irqsave(¤t_lb->lock, flags); + dev = current_lb->find_device(current_lb, ci, data); + spin_unlock_irqrestore(¤t_lb->lock, flags); + + return dev; +} + +static int __acrypto_lb_register(struct acrypto_lb *lb) +{ + int err; + + spin_lock_init(&lb->lock); + + init_completion(&lb->dev_released); + memcpy(&lb->device, &acrypto_lb_dev, sizeof(struct device)); + lb->driver = &acrypto_lb_driver; + + snprintf(lb->device.bus_id, sizeof(lb->device.bus_id), "%s", lb->name); + err = device_register(&lb->device); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto load balancer device %s: err=%d.\n", + lb->name, err); + return err; + } + + snprintf(lb->class_device.class_id, sizeof(lb->class_device.class_id), "%s", lb->name); + lb->class_device.dev = &lb->device; + lb->class_device.class = &acrypto_lb_class; + + err = class_device_register(&lb->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto load balancer class device %s: err=%d.\n", + lb->name, err); + device_unregister(&lb->device); + return err; + } + + create_device_attributes(lb); + + return 0; + +} + +void __acrypto_lb_unregister(struct acrypto_lb *lb) +{ + remove_device_attributes(lb); + class_device_unregister(&lb->class_device); + device_unregister(&lb->device); +} + +int acrypto_lb_register(struct acrypto_lb *lb, int set_current, int set_default) +{ + struct acrypto_lb *__lb; + int err; + unsigned long flags; + + spin_lock_irqsave(&acrypto_lb_lock, flags); + + list_for_each_entry(__lb, &acrypto_lb_list, lb_entry) { + if (unlikely(compare_lb(__lb, lb))) { + spin_unlock_irqrestore(&acrypto_lb_lock, flags); + + dprintk(KERN_ERR "Crypto load balancer %s is already registered.\n", + lb->name); + return -EINVAL; + } + } + + list_add(&lb->lb_entry, &acrypto_lb_list); + + if (!default_lb || set_default) + default_lb = lb; + + if (!current_lb || set_current) + current_lb = lb; + + spin_unlock_irqrestore(&acrypto_lb_lock, flags); + + err = __acrypto_lb_register(lb); + if (err) { + spin_lock_irqsave(&acrypto_lb_lock, flags); + list_del_init(&lb->lb_entry); + current_lb = &simple_lb; + default_lb = &simple_lb; + spin_unlock_irqrestore(&acrypto_lb_lock, flags); + + return err; + } + + dprintk("Crypto load balancer %s was registered and set to be [%s.%s].\n", + lb->name, (lb_is_current(lb)) ? "current" : "not current", + (lb_is_default(lb)) ? "default" : "not default"); + + lb_num++; + + return 0; +} + +void acrypto_lb_unregister(struct acrypto_lb *lb) +{ + struct acrypto_lb *__lb, *n; + unsigned long flags; + + __acrypto_lb_unregister(lb); + + spin_lock_irqsave(&acrypto_lb_lock, flags); + + list_for_each_entry_safe(__lb, n, &acrypto_lb_list, lb_entry) { + if (compare_lb(__lb, lb)) { + lb_num--; + list_del_init(&__lb->lb_entry); + + dprintk("Crypto load balancer %s was unregistered.\n", lb->name); + break; + } + } + + spin_unlock_irqrestore(&acrypto_lb_lock, flags); +} + +static void acrypto_lb_queue_wrapper(void *data) +{ + struct acrypto_session *s = (struct acrypto_session *)data; + + dprintk("%s: Calling callback for session %08x [%08x] flags=%lx, " + "op=%04u, type=%04x, mode=%04x, priority=%04x\n", __func__, + s->ci.id, s->ci.dev_id, s->ci.flags, s->ci.operation, + s->ci.type, s->ci.mode, s->ci.priority); + + s->ci.callback(&s->ci, &s->data); + + if (likely(session_completed(s))) { + acrypto_session_destroy(s); + return; + } else { + /* + * Not implemented yet. + */ + /* + * Special case: acrypto consumer marks session as "not finished" + * in it's callback - it means that acrypto consumer wants + * this session to be processed further, + * for example acrypto consumer can add new route and then + * mark session as "not finished". + */ + BUG(); + } +} + +static void acrypto_lb_process_next_route(struct acrypto_session *s) +{ + struct acrypto_device *dev; + + dev = acrypto_route_copy_current_ci(s); + if (dev) + __acrypto_session_add(dev, s); +} + +int acrypto_complete_session(struct acrypto_session *s) +{ + struct acrypto_device *cdev; + + cdev = acrypto_route_get_current_device(s); + + acrypto_stat_complete_inc(s); + acrypto_stat_ptime_inc(s); + acrypto_session_dequeue_route(s); + set_bit(SESSION_COMPLETED, &s->ci.flags); + + if (session_broken(s)) { + struct acrypto_device *ldev; + int err; + + unbreak_session(s); + ldev = acrypto_lb_find_device(&s->ci, &s->data); + if (ldev && ldev != cdev) { + err = acrypto_route_add_direct(ldev, s, &s->ci); + if (!err) { + err = acrypto_session_add(s); + if (!err) + return 0; + } + } else if (ldev) + acrypto_device_put(ldev); + break_session(s); + } + + if (unlikely(acrypto_route_queue_len(s))) { + acrypto_lb_process_next_route(s); + } else { + if (test_bit(SESSION_DIRECT, &s->ci.flags)) + acrypto_lb_queue_wrapper(s); + else { + INIT_WORK(&s->work, &acrypto_lb_queue_wrapper, s); + queue_work(acrypto_lb_queue, &s->work); + } + } + + return 0; +} + +int acrypto_lb_init(void) +{ + int err; + + err = bus_register(&acrypto_lb_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto load balancer bus: err=%d.\n", err); + goto err_out_exit; + } + + err = driver_register(&acrypto_lb_driver); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto load balancer driver: err=%d.\n", err); + goto err_out_bus_unregister; + } + + acrypto_lb_class.class_dev_attrs = &class_device_attr_lbs; + + err = class_register(&acrypto_lb_class); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto load balancer class: err=%d.\n", err); + goto err_out_driver_unregister; + } + + acrypto_lb_queue = create_singlethread_workqueue("clbq"); + if (!acrypto_lb_queue) { + dprintk(KERN_ERR "Failed to create acrypto load balaner work queue.\n"); + goto err_out_class_unregister; + } + + err = acrypto_lb_register(&simple_lb, 1, 1); + if (err) + goto err_out_destroy_workqueue; + + return 0; + +err_out_destroy_workqueue: + destroy_workqueue(acrypto_lb_queue); +err_out_class_unregister: + class_unregister(&acrypto_lb_class); +err_out_driver_unregister: + driver_unregister(&acrypto_lb_driver); +err_out_bus_unregister: + bus_unregister(&acrypto_lb_bus_type); +err_out_exit: + return err; +} + +void acrypto_lb_fini(void) +{ + __acrypto_lb_unregister(&simple_lb); + + flush_workqueue(acrypto_lb_queue); + destroy_workqueue(acrypto_lb_queue); + class_unregister(&acrypto_lb_class); + driver_unregister(&acrypto_lb_driver); + bus_unregister(&acrypto_lb_bus_type); +} + +EXPORT_SYMBOL_GPL(acrypto_lb_register); +EXPORT_SYMBOL_GPL(acrypto_lb_unregister); +EXPORT_SYMBOL_GPL(acrypto_lb_find_device); +EXPORT_SYMBOL_GPL(acrypto_complete_session); diff --git a/acrypto/acrypto_lb.h b/acrypto/acrypto_lb.h new file mode 100644 index 0000000..d65e17c --- /dev/null +++ b/acrypto/acrypto_lb.h @@ -0,0 +1,56 @@ +/* + * acrypto_lb.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_LB_H +#define __ACRYPTO_LB_H + +#include + +#define ACRYPTO_LB_NAMELEN 32 + +struct acrypto_lb +{ + struct list_head lb_entry; + + char name[ACRYPTO_LB_NAMELEN]; + + struct acrypto_device * (*find_device) (struct acrypto_lb *, + struct acrypto_session_initializer *, + struct acrypto_data *); + + spinlock_t lock; + + struct device_driver *driver; + struct device device; + struct class_device class_device; + struct completion dev_released; + +}; + +int acrypto_lb_register(struct acrypto_lb *lb, int set_current, int set_default); +void acrypto_lb_unregister(struct acrypto_lb *); + +struct acrypto_device *acrypto_lb_find_device(struct acrypto_session_initializer *, struct acrypto_data *); + +int acrypto_lb_init(void); +void acrypto_lb_fini(void); + +#endif /* __ACRYPTO_LB_H */ diff --git a/acrypto/acrypto_main.c b/acrypto/acrypto_main.c new file mode 100644 index 0000000..e66a63e --- /dev/null +++ b/acrypto/acrypto_main.c @@ -0,0 +1,477 @@ +/* + * acrypto_main.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "acrypto_lb.h" +#include "acrypto_conn.h" +#include "acrypto_route.h" +#include "acrypto_user_ioctl.h" +#include "acrypto_stat.h" + +unsigned int max_ca_drivers = 16; /* max # of context-aware drivers */ +module_param(max_ca_drivers, uint, 0); + +extern struct bus_type acrypto_bus_type; +extern struct device_driver acrypto_driver; +extern struct class acrypto_class; +extern struct device acrypto_dev; +extern spinlock_t cdev_lock; +extern struct list_head cdev_list; + +static atomic_t global_session_id = ATOMIC_INIT(0); +static int acrypto_use_connector; +static DEFINE_MUTEX(context_mutex); +static LIST_HEAD(context_list); + +void acrypto_session_destroy(struct acrypto_session *s) +{ + if (s->data.priv_size && s->data.priv) + kfree(s->data.priv); + + if (session_from_cache(s)) + kmem_cache_free(s->pool_dev->session_cache, s); + else + mempool_free(s, s->pool_dev->session_pool); +} + +struct acrypto_session *acrypto_session_create(struct acrypto_session_initializer *ci, struct acrypto_data *d) +{ + struct acrypto_device *ldev; + struct acrypto_session *s; + int err; + + if (d->priv_size > ACRYPTO_MAX_PRIV_SIZE) { + dprintk("priv_size %u is too big, maximum allowed %u.\n", + d->priv_size, ACRYPTO_MAX_PRIV_SIZE); + return NULL; + } + + /* + * We grab a reference here if we can find device. + * This reference will be dropped when route will be removed. + */ + ldev = acrypto_lb_find_device(ci, d); + if (!ldev) { + dprintk("Cannot find suitable device for [%02x.%02x.%02x.%02x].\n", + ci->operation, ci->mode, ci->type, ci->priority); + return NULL; + } + + s = mempool_alloc(ldev->session_pool, GFP_ATOMIC); + if (!s) { + ldev->stat.pool_failed++; + + s = kmem_cache_alloc(ldev->session_cache, GFP_ATOMIC); + if (!s) { + ldev->stat.kmem_failed++; + goto err_out_device_put; + } + + mark_session_from_cache(s); + } + + s->pool_dev = ldev; + + acrypto_route_head_init(&s->route_list); + INIT_LIST_HEAD(&s->dev_queue_entry); + + spin_lock_init(&s->lock); + + memcpy(&s->ci, ci, sizeof(s->ci)); + memcpy(&s->data, d, sizeof(s->data)); + + s->data.priv = NULL; + if (d->priv_size) { + s->data.priv = kmalloc(d->priv_size, GFP_ATOMIC); + if (!s->data.priv) { + ldev->stat.kmem_failed++; + goto err_out_session_free; + } + + if (d->priv) + memcpy(s->data.priv, d->priv, d->priv_size); + } + else + s->data.priv = d->priv; + + s->ci.id = atomic_inc_return(&global_session_id); + s->ci.dev_id = atomic_inc_return(&ldev->sid); + s->ci.flags = 0; + + err = acrypto_route_add_direct(ldev, s, ci); + if (err) { + ldev->stat.route_failed++; + dprintk("Can not add route to device %s.\n", ldev->name); + goto err_out_session_free; + } + + return s; + +err_out_session_free: + acrypto_session_destroy(s); +err_out_device_put: + acrypto_device_put(ldev); + + return NULL; +} + +int __acrypto_session_add(struct acrypto_device *ldev, struct acrypto_session *s) +{ + int err; + + err = acrypto_session_enqueue(ldev, s); + + if (!err && ldev->data_ready) + ldev->data_ready(ldev); + + return err; +} + +int acrypto_session_add(struct acrypto_session *s) +{ + struct acrypto_device *ldev; + int err; + + ldev = acrypto_route_get_current_device(s); + if (!ldev) + return -ENODEV; + + err = __acrypto_session_add(ldev, s); + if (err) + /* + * Decrement usage counter, which was gotten in lookup() method. + */ + acrypto_device_put(ldev); + + return err; +} + +struct acrypto_session *acrypto_session_alloc(struct acrypto_session_initializer *ci, struct acrypto_data *d) +{ + struct acrypto_session *s; + int err; + + s = acrypto_session_create(ci, d); + if (!s) + return NULL; + + err = acrypto_session_add(s); + if (err) { + acrypto_session_destroy(s); + return NULL; + } + + return s; +} + +int acrypto_session_dequeue_route(struct acrypto_session *s) +{ + struct acrypto_route *rt; + + rt = acrypto_route_dequeue(s); + if (rt) { + acrypto_route_free(rt); + return 0; + } + + return -ENODEV; +} + +void acrypto_process(struct acrypto_session *s, + int (*proc)(struct acrypto_session *s, void *src, int slen, void *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + int (*proc_scatter)(struct acrypto_session *s, struct scatterlist *src, + struct scatterlist *dst, int nbytes, void *key, int klen, void *iv, int ilen, void *data), + void (*finish)(struct acrypto_session *s, void *key, int klen, void *iv, int ilen, void *data), + void *data) +{ + int i, keylen, ivlen, err = -EINVAL; + void *key, *iv; + + if (session_completed(s)) + return; + + key = iv = NULL; + ivlen = keylen = 0; + + err = -ENOMEM; + /* + * Simple case - key is small(it's size is less than PAGE_SIZE). + * Assymetric acrypto will require proper key sg handling. + */ + if (s->data.sg_key[0].length) { + key = kmap(s->data.sg_key[0].page); + if (!key) + goto err_out_exit; + key += s->data.sg_key[0].offset; + keylen = s->data.sg_key[0].length; + } + + if (s->data.sg_iv[0].length) { + iv = kmap(s->data.sg_iv[0].page); + if (!iv) + goto err_out_unmap_key; + iv += s->data.sg_iv[0].offset; + ivlen = s->data.sg_iv[0].length; + } + + if (proc) { + for (i=0; idata.sg_src_num; ++i) { + int dlen, slen; + u8 *dst, *src; + + slen = s->data.sg_src[i].length; + dlen = s->data.sg_dst[i].length; + + + dst = kmap_atomic(s->data.sg_dst[i].page, KM_USER0) + s->data.sg_dst[i].offset; + src = kmap_atomic(s->data.sg_src[i].page, KM_USER1) + s->data.sg_src[i].offset; + + err = proc(s, src, slen, dst, dlen, key, keylen, iv, ivlen, data); + + kunmap_atomic(src, KM_USER1); + kunmap_atomic(dst, KM_USER0); + + s->data.sg_dst[i].length = s->data.sg_src[i].length; + s->data.sg_dst[i].offset = s->data.sg_src[i].offset; + } + } else if (proc_scatter) { + int nbytes = 0; + + for (i=0; idata.sg_src_num; ++i) + nbytes += s->data.sg_src[i].length; + err = proc_scatter(s, s->data.sg_src, s->data.sg_dst, nbytes, key, keylen, iv, ivlen, data); + } else + err = -EINVAL; + + if (err < 0) { + break_session(s); + if (printk_ratelimit()) + printk("acrypto: operation=%02x, proc=%p, proc_scatter=%p, err=%d.\n", + s->ci.operation, proc, proc_scatter, err); + } + + if (finish) + finish(s, key, keylen, iv, ivlen, data); + + if (iv) + kunmap(s->data.sg_iv[0].page); +err_out_unmap_key: + if (key) + kunmap(s->data.sg_key[0].page); +err_out_exit: + + acrypto_complete_session(s); +} + +int acrypto_context_register(struct acrypto_context *ctx) +{ + unsigned long flags; + struct acrypto_device *dev; + int num_succeeded = 0; + + mutex_lock(&context_mutex); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->flags & ACRYPTO_DEV_CTX_INIT) { + if (dev->context_callback(dev, ctx, ACRYPTO_DEV_CTX_INIT) == 0) + num_succeeded++; + } else /* context-unaware-drivers can't fail here */ + num_succeeded++; + } + spin_unlock_irqrestore(&cdev_lock, flags); + if (num_succeeded) + list_add_tail(&ctx->entry, &context_list); + mutex_unlock(&context_mutex); + + return (num_succeeded ? 0 : -EINVAL); +} + +void acrypto_context_unregister(struct acrypto_context *ctx) +{ + unsigned long flags; + struct acrypto_device *dev; + + mutex_lock(&context_mutex); + list_add_tail(&ctx->entry, &context_list); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->flags & ACRYPTO_DEV_CTX_EXIT) + dev->context_callback(dev, ctx, ACRYPTO_DEV_CTX_EXIT); + } + spin_unlock_irqrestore(&cdev_lock, flags); + mutex_unlock(&context_mutex); +} + +void acrypto_context_notify(struct acrypto_context *ctx, u32 ctx_flags) +{ + unsigned long flags; + struct acrypto_device *dev; + + mutex_lock(&context_mutex); + list_add_tail(&ctx->entry, &context_list); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if ((dev->flags & ctx_flags) && dev->context_callback) + dev->context_callback(dev, ctx, ctx_flags); + } + spin_unlock_irqrestore(&cdev_lock, flags); + mutex_unlock(&context_mutex); +} + +struct acrypto_context *acrypto_context_alloc(u16 type, u16 mode, u8 *key, unsigned int key_size) +{ + struct acrypto_context *ctx; + int err; + int driver_priv_size; + + driver_priv_size = sizeof(void*) * max_ca_drivers; + ctx = kzalloc(sizeof(struct acrypto_context) + driver_priv_size, GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->type = type; + ctx->mode = mode; + ctx->key_size = key_size; + + ctx->key = kmalloc(key_size, GFP_KERNEL); + if (!ctx->key) { + kfree(ctx); + return NULL; + } + memcpy(ctx->key, key, key_size); + atomic_set(&ctx->refcnt, 1); + + err = acrypto_context_register(ctx); + if (err) { + kfree(ctx->key); + kfree(ctx); + return NULL; + } + + return ctx; +} + +void acrypto_context_free(struct acrypto_context *ctx) +{ + acrypto_context_unregister(ctx); + kfree(ctx->key); + kfree(ctx); +} + +int cmain_init(void) +{ + int err; + + err = bus_register(&acrypto_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto bus: err=%d.\n", + err); + return err; + } + + err = driver_register(&acrypto_driver); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto driver: err=%d.\n", + err); + goto err_out_bus_unregister; + } + + err = class_register(&acrypto_class); + if (err) { + dprintk(KERN_ERR "Failed to register acrypto class: err=%d.\n", + err); + goto err_out_driver_unregister; + } + + err = acrypto_lb_init(); + if (err) + goto err_out_class_unregister; + + acrypto_use_connector = 0; + err = acrypto_conn_init(); + if (!err) + acrypto_use_connector = 1; + if (!acrypto_use_connector) + printk(KERN_INFO "%s: Connector has not initialized yet, support disabled.\n", __func__); + + err = acrypto_user_ioctl_init(); + if (err) + goto err_out_acrypto_conn_fini; + + return 0; + +err_out_acrypto_conn_fini: + if (acrypto_use_connector) + acrypto_conn_fini(); + acrypto_lb_fini(); +err_out_class_unregister: + class_unregister(&acrypto_class); +err_out_driver_unregister: + driver_unregister(&acrypto_driver); +err_out_bus_unregister: + bus_unregister(&acrypto_bus_type); + + return err; +} + +void cmain_fini(void) +{ + acrypto_user_ioctl_fini(); + + if (acrypto_use_connector) + acrypto_conn_fini(); + acrypto_lb_fini(); + + class_unregister(&acrypto_class); + driver_unregister(&acrypto_driver); + bus_unregister(&acrypto_bus_type); +} + +module_init(cmain_init); +module_exit(cmain_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Asynchronous acrypto layer."); + +EXPORT_SYMBOL(acrypto_session_alloc); +EXPORT_SYMBOL_GPL(acrypto_session_create); +EXPORT_SYMBOL_GPL(acrypto_session_add); +EXPORT_SYMBOL_GPL(__acrypto_session_add); +EXPORT_SYMBOL_GPL(acrypto_process); +EXPORT_SYMBOL_GPL(acrypto_session_dequeue_route); +EXPORT_SYMBOL_GPL(acrypto_context_alloc); +EXPORT_SYMBOL_GPL(acrypto_context_free); +EXPORT_SYMBOL_GPL(acrypto_context_register); +EXPORT_SYMBOL_GPL(acrypto_context_unregister); +EXPORT_SYMBOL_GPL(acrypto_context_notify); diff --git a/acrypto/acrypto_queue.c b/acrypto/acrypto_queue.c new file mode 100644 index 0000000..15f4ff2 --- /dev/null +++ b/acrypto/acrypto_queue.c @@ -0,0 +1,168 @@ +/* + * acrypto_queue.c + * + * Copyright (c) 2005 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static struct acrypto_session *acrypto_dequeue_one(struct acrypto_queue *q) +{ + struct acrypto_session *s = NULL; + + if (q->qlen) { + s = list_entry(q->session_list.next, struct acrypto_session, dev_queue_entry); + list_del(&s->dev_queue_entry); + q->qlen--; + } + + return s; +} + +static int acrypto_enqueue_one(struct acrypto_queue *q, struct acrypto_session *s) +{ + list_add_tail(&s->dev_queue_entry, &q->session_list); + q->qlen++; + + return 0; +} + +static struct acrypto_queue *acrypto_create_queue(struct acrypto_device *dev, u16 priority) +{ + struct acrypto_queue *q, *n; + struct list_head *prev; + int found = 0; + + q = mempool_alloc(dev->queue_pool, GFP_ATOMIC); + if (!q) + return NULL; + + INIT_LIST_HEAD(&q->session_list); + q->qlen = 0; + q->priority = priority; + + prev = &dev->queue.queue_list; + list_for_each_entry(n, &dev->queue.queue_list, queue_entry) { + if (n->priority > priority) { + list_add(&q->queue_entry, prev); + dev->queue.num++; + found = 1; + break; + } + prev = &n->queue_entry; + } + + if (!found) { + list_add(&q->queue_entry, &dev->queue.queue_list); + dev->queue.num++; + return q; + } + + + return q; +} + +static void acrypto_queue_flush(struct acrypto_queue *q) +{ + struct acrypto_session *s; + + while ((s = acrypto_dequeue_one(q)) != NULL) + acrypto_session_destroy(s); +} + +void acrypto_destroy_queues(struct acrypto_device *dev) +{ + unsigned long flags; + struct acrypto_queue *q, *n; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + list_for_each_entry_safe(q, n, &dev->queue.queue_list, queue_entry) { + list_del(&q->queue_entry); + dev->queue.num--; + + acrypto_queue_flush(q); + + mempool_free(q, dev->queue_pool); + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); +} + +int acrypto_session_enqueue(struct acrypto_device *dev, struct acrypto_session *s) +{ + int err = -EINVAL, found = 0; + unsigned long flags; + struct acrypto_queue *q; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + list_for_each_entry(q, &dev->queue.queue_list, queue_entry) { + if (q->priority == s->ci.priority) { + err = acrypto_enqueue_one(q, s); + found = 1; + break; + } + } + + if (!found) { + q = acrypto_create_queue(dev, s->ci.priority); + if (q) + err = acrypto_enqueue_one(q, s); + else + err = -ENOMEM; + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); + + return err; +} + +struct acrypto_session *acrypto_session_dequeue(struct acrypto_device *dev) +{ + unsigned long flags; + struct acrypto_session *s = NULL; + struct acrypto_queue *q; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + if (dev->queue.num) { + list_for_each_entry(q, &dev->queue.queue_list, queue_entry) { + if (q->qlen) { + s = acrypto_dequeue_one(q); + if (s) + break; + } + } + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); + + return s; +} + +EXPORT_SYMBOL_GPL(acrypto_session_dequeue); +EXPORT_SYMBOL_GPL(acrypto_destroy_queues); diff --git a/acrypto/acrypto_route.h b/acrypto/acrypto_route.h new file mode 100644 index 0000000..761a723 --- /dev/null +++ b/acrypto/acrypto_route.h @@ -0,0 +1,273 @@ +/* + * acrypto_route.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_ROUTE_H +#define __ACRYPTO_ROUTE_H + +#include +#include +#include + +#include + +static inline struct acrypto_route *acrypto_route_alloc_direct(struct acrypto_device *dev, + struct acrypto_session_initializer *ci) +{ + struct acrypto_route *rt; + + rt = mempool_alloc(dev->route_pool, GFP_ATOMIC); + if (!rt) + return NULL; + + memset(rt, 0, sizeof(*rt)); + memcpy(&rt->ci, ci, sizeof(*ci)); + + rt->dev = dev; + + return rt; +} + +static inline struct acrypto_route *acrypto_route_alloc(struct acrypto_device *dev, + struct acrypto_session_initializer *ci) +{ + struct acrypto_route *rt; + + if (!match_initializer(dev, ci)) + return NULL; + + rt = acrypto_route_alloc_direct(dev, ci); + + return rt; +} + +/* + * Device's reference counter must be incremented before routing was + * allocated, so routing subsystem itself does not care about it. + */ +static inline void acrypto_route_free(struct acrypto_route *rt) +{ + struct acrypto_device *dev = rt->dev; + + mempool_free(rt, dev->route_pool); + acrypto_device_put(dev); +} + +static inline void __acrypto_route_del(struct acrypto_route *rt, struct acrypto_route_head *list) +{ + struct acrypto_route *next, *prev; + + list->qlen--; + next = rt->next; + prev = rt->prev; + rt->next = rt->prev = NULL; + rt->list = NULL; + next->prev = prev; + prev->next = next; +} + +static inline void acrypto_route_del(struct acrypto_route *rt) +{ + struct acrypto_route_head *list = rt->list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + if (list == rt->list) + __acrypto_route_del(rt, rt->list); + spin_unlock_irqrestore(&list->lock, flags); + + acrypto_route_free(rt); + } +} + +static inline struct acrypto_route *__acrypto_route_dequeue(struct acrypto_route_head *list) +{ + struct acrypto_route *next, *prev, *result; + + prev = (struct acrypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) { + result = next; + next = next->next; + list->qlen--; + next->prev = prev; + prev->next = next; + result->next = result->prev = NULL; + result->list = NULL; + } + return result; +} + +static inline struct acrypto_route *acrypto_route_dequeue(struct acrypto_session *s) +{ + struct acrypto_route *rt; + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + rt = __acrypto_route_dequeue(&s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); + + return rt; +} + +static inline void __acrypto_route_queue(struct acrypto_route *rt, struct acrypto_route_head *list) +{ + struct acrypto_route *prev, *next; + + rt->list = list; + list->qlen++; + next = (struct acrypto_route *)list; + prev = next->prev; + rt->next = next; + rt->prev = prev; + next->prev = prev->next = rt; +} + +static inline void acrypto_route_queue(struct acrypto_route *rt, struct acrypto_session *s) +{ + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + __acrypto_route_queue(rt, &s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); +} + +/* + * Caller must hold a reference for acrypto_device being added, + * and it is not permitted to drop the reference until routing is removed. + * See acrypto_session_create() for example of right device's reference counter usage. + */ + +static inline int acrypto_route_add(struct acrypto_device *dev, struct acrypto_session *s, + struct acrypto_session_initializer *ci) +{ + struct acrypto_route *rt; + + rt = acrypto_route_alloc(dev, ci); + if (!rt) + return -ENOMEM; + + acrypto_route_queue(rt, s); + + return 0; +} + +static inline int acrypto_route_add_direct(struct acrypto_device *dev, struct acrypto_session *s, + struct acrypto_session_initializer *ci) +{ + struct acrypto_route *rt; + + rt = acrypto_route_alloc_direct(dev, ci); + if (!rt) + return -ENOMEM; + + acrypto_route_queue(rt, s); + + return 0; +} + +static inline int acrypto_route_queue_len(struct acrypto_session *s) +{ + return s->route_list.qlen; +} + +static inline void acrypto_route_head_init(struct acrypto_route_head *list) +{ + spin_lock_init(&list->lock); + list->prev = list->next = (struct acrypto_route *)list; + list->qlen = 0; +} + +static inline struct acrypto_route *__acrypto_route_current(struct acrypto_route_head *list) +{ + struct acrypto_route *next, *prev, *result; + + prev = (struct acrypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) + result = next; + + return result; +} + +static inline struct acrypto_route *acrypto_route_current(struct acrypto_session *s) +{ + struct acrypto_route_head *list; + struct acrypto_route *rt = NULL; + + list = &s->route_list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __acrypto_route_current(list); + spin_unlock_irqrestore(&list->lock, flags); + } + + return rt; +} + +static inline struct acrypto_device *acrypto_route_copy_current_ci(struct acrypto_session *s) +{ + struct acrypto_route *rt; + struct acrypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + + rt = __acrypto_route_current(list); + if (rt) { + memcpy(&s->ci, &rt->ci, sizeof(s->ci)); + return rt->dev; + } + + spin_unlock_irqrestore(&list->lock, flags); + + return NULL; +} + +/* + * It is not allowed to use returned device pointer + * after current route is removed. + * If you want to use current device and remove current route, + * you MUST grab the reference before removing the route. + */ +static inline struct acrypto_device *acrypto_route_get_current_device(struct acrypto_session *s) +{ + struct acrypto_route *rt; + struct acrypto_device *dev = NULL; + struct acrypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __acrypto_route_current(list); + if (rt) + dev = rt->dev; + spin_unlock_irqrestore(&list->lock, flags); + + return dev; +} + +#endif /* __ACRYPTO_ROUTE_H */ diff --git a/acrypto/acrypto_stat.c b/acrypto/acrypto_stat.c new file mode 100644 index 0000000..730fc90 --- /dev/null +++ b/acrypto/acrypto_stat.c @@ -0,0 +1,69 @@ +/* + * acrypto_stat.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "acrypto_route.h" + +void acrypto_stat_complete_inc(struct acrypto_session *s) +{ + struct acrypto_device *dev; + + dev = acrypto_route_get_current_device(s); + if (dev) { + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + dev->stat.scompleted++; + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +void acrypto_stat_ptime_inc(struct acrypto_session *s) +{ + struct acrypto_device *dev; + + dev = acrypto_route_get_current_device(s); + if (dev) { + int i; + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + for (i = 0; i < dev->cap_number; ++i) { + if (__match_initializer(&dev->cap[i], &s->ci, dev)) { + dev->cap[i].ptime += s->ci.ptime; + dev->cap[i].scomp++; + break; + } + } + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +EXPORT_SYMBOL(acrypto_stat_complete_inc); diff --git a/acrypto/acrypto_stat.h b/acrypto/acrypto_stat.h new file mode 100644 index 0000000..66f9dd7 --- /dev/null +++ b/acrypto/acrypto_stat.h @@ -0,0 +1,30 @@ +/* + * acrypto_stat.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_STAT_H +#define __ACRYPTO_STAT_H + +#include + +void acrypto_stat_complete_inc(struct acrypto_session *s); +void acrypto_stat_ptime_inc(struct acrypto_session *s); + +#endif /* __ACRYPTO_STAT_H */ diff --git a/acrypto/acrypto_user.c b/acrypto/acrypto_user.c new file mode 100644 index 0000000..3c5741b --- /dev/null +++ b/acrypto/acrypto_user.c @@ -0,0 +1,196 @@ +/* + * acrypto_user.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "acrypto_user.h" + +static struct scatterlist *acrypto_user_alloc_sg(int sg_num) +{ + struct scatterlist *sg; + + sg = kmalloc(sg_num * sizeof(*sg), GFP_ATOMIC); + if (!sg) { + dprintk("Failed to allocate %d scatterlist structures.\n", sg_num); + return NULL; + } + + memset(sg, 0, sizeof(*sg) * sg_num); + + return sg; +} + +static void acrypto_user_free_sg(struct scatterlist *sg) +{ + kfree(sg); +} + +int acrypto_user_alloc_acrypto_data(struct acrypto_data *data, int src_size, int dst_size, int key_size, int iv_size) +{ + int sg_num; + + if ( (src_size > MAX_DATA_SIZE * PAGE_SIZE) || + (dst_size > MAX_DATA_SIZE * PAGE_SIZE) || + (key_size > MAX_DATA_SIZE * PAGE_SIZE) || + (iv_size > MAX_DATA_SIZE * PAGE_SIZE)) { + dprintk("Sizes are too big: src=%u, dst=%u, key=%u, iv=%u, max=%u.\n", + src_size, dst_size, key_size, iv_size, MAX_DATA_SIZE); + return -EINVAL; + + } + + sg_num = ALIGN_DATA_SIZE(src_size) / PAGE_SIZE; + data->sg_src = acrypto_user_alloc_sg(sg_num); + if (!data->sg_src) + goto err_out_exit; + data->sg_src_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(dst_size) / PAGE_SIZE; + data->sg_dst = acrypto_user_alloc_sg(sg_num); + if (!data->sg_dst) + goto err_out_free_src; + data->sg_dst_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(key_size) / PAGE_SIZE; + data->sg_key = acrypto_user_alloc_sg(sg_num); + if (!data->sg_key) + goto err_out_free_dst; + data->sg_key_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(iv_size) / PAGE_SIZE; + data->sg_iv = acrypto_user_alloc_sg(sg_num); + if (!data->sg_iv) + goto err_out_free_key; + data->sg_iv_num = sg_num; + + return 0; + +err_out_free_key: + acrypto_user_free_sg(data->sg_key); +err_out_free_dst: + acrypto_user_free_sg(data->sg_dst); +err_out_free_src: + acrypto_user_free_sg(data->sg_src); +err_out_exit: + + return -ENOMEM; +} + +void acrypto_user_free_acrypto_data(struct acrypto_data *data) +{ + acrypto_user_free_sg(data->sg_src); + acrypto_user_free_sg(data->sg_dst); + acrypto_user_free_sg(data->sg_key); + acrypto_user_free_sg(data->sg_iv); +} + +void acrypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg) +{ + int i, sg_num; + + sg_num = ALIGN_DATA_SIZE(size) / PAGE_SIZE; + + dprintk("Filling %d sgs, total size %u: ", sg_num, size); + + for (i=0; i size) + sg[i].length = size; + } else { + sg[i].offset = 0; + sg[i].length = (i != sg_num-1)?PAGE_SIZE:size; + } + dprintka("%x.%x.%p.%lx ", sg[i].offset, sg[i].length, ptr, ALIGN_DATA_SIZE((unsigned long)ptr)); + + size -= sg[i].length; + ptr += sg[i].length; + } + dprintka("\n"); +} + +struct scatterlist *acrypto_user_get_sg(struct acrypto_user_data *ud, struct acrypto_data *data) +{ + struct scatterlist *sg = NULL; + int inval = 0; + + switch (ud->data_type) { + case ACRYPTO_USER_DATA_SRC: + sg = data->sg_src; + inval = (data->sg_src_num * PAGE_SIZE < ud->data_size); + dprintk("Found SRC data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case ACRYPTO_USER_DATA_DST: + sg = data->sg_dst; + inval = (data->sg_dst_num * PAGE_SIZE < ud->data_size); + dprintk("Found DST data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case ACRYPTO_USER_DATA_KEY: + sg = data->sg_key; + inval = (data->sg_key_num * PAGE_SIZE < ud->data_size); + dprintk("Found KEY data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case ACRYPTO_USER_DATA_IV: + sg = data->sg_iv; + inval = (data->sg_iv_num * PAGE_SIZE < ud->data_size); + dprintk("Found IV data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + default: + dprintk("Unknown data type 0x%x, size=%u.\n", ud->data_type, ud->data_size); + break; + } + + return (inval)?NULL:sg; +} + +int acrypto_user_fill_sg_data(struct acrypto_user_data *ud, struct acrypto_data *data, void *ptr) +{ + struct scatterlist *sg; + + sg = acrypto_user_get_sg(ud, data); + if (!sg) + return -EINVAL; + + acrypto_user_fill_sg(ptr, ud->data_size, sg); + + return 0; +} + +EXPORT_SYMBOL_GPL(acrypto_user_alloc_acrypto_data); +EXPORT_SYMBOL_GPL(acrypto_user_free_acrypto_data); +EXPORT_SYMBOL_GPL(acrypto_user_fill_sg); +EXPORT_SYMBOL_GPL(acrypto_user_fill_sg_data); +EXPORT_SYMBOL_GPL(acrypto_user_get_sg); diff --git a/acrypto/acrypto_user.h b/acrypto/acrypto_user.h new file mode 100644 index 0000000..168a1a9 --- /dev/null +++ b/acrypto/acrypto_user.h @@ -0,0 +1,52 @@ +/* + * acrypto_user.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_USER_H +#define __ACRYPTO_USER_H + +#define MAX_DATA_SIZE 3 +#define ALIGN_DATA_SIZE(size) ((size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) + +enum acrypto_user_data_types +{ + ACRYPTO_USER_DATA_SRC = 0, + ACRYPTO_USER_DATA_DST, + ACRYPTO_USER_DATA_KEY, + ACRYPTO_USER_DATA_IV, +}; + +struct acrypto_user_data +{ + __u16 data_size; + __u16 data_type; +}; + + +#ifdef __KERNEL__ + +int acrypto_user_alloc_acrypto_data(struct acrypto_data *data, int src_size, int dst_size, int key_size, int iv_size); +void acrypto_user_free_acrypto_data(struct acrypto_data *data); +void acrypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg); +struct scatterlist *acrypto_user_get_sg(struct acrypto_user_data *ud, struct acrypto_data *data); +int acrypto_user_fill_sg_data(struct acrypto_user_data *ud, struct acrypto_data *data, void *ptr); + +#endif /* __KERNEL__ */ +#endif /* __ACRYPTO_USER_H */ diff --git a/acrypto/acrypto_user_direct.c b/acrypto/acrypto_user_direct.c new file mode 100644 index 0000000..0c2e0ef --- /dev/null +++ b/acrypto/acrypto_user_direct.c @@ -0,0 +1,391 @@ +/* + * acrypto_user_direct.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "acrypto_user.h" +#include "acrypto_user_direct.h" + +extern struct cb_id acrypto_conn_id; + +static LIST_HEAD(acrypto_user_direct_list); +static spinlock_t acrypto_user_direct_lock = SPIN_LOCK_UNLOCKED; +static struct completion thread_exited; +static int need_exit; +static DECLARE_WAIT_QUEUE_HEAD(acrypto_user_direct_wait_queue); +static int acrypto_user_direct_has_work; + +static int acrypto_user_direct_alloc_pages(struct acrypto_user_direct_kern *k) +{ + k->sp = kmalloc(sizeof(struct page *) * k->snum, GFP_KERNEL); + if (!k->sp) { + dprintk("Failed to allocate %d source pages.\n", k->snum); + return -ENOMEM; + } + + k->dp = kmalloc(sizeof(struct page *) * k->dnum, GFP_KERNEL); + if (!k->dp) { + dprintk("Failed to allocate %d destination pages.\n", k->dnum); + kfree(k->sp); + return -ENOMEM; + } + + return 0; +} + +static void acrypto_user_direct_free_pages(struct acrypto_user_direct_kern *k) +{ + kfree(k->sp); + kfree(k->dp); +} + +static int acrypto_user_direct_alloc_vmas(struct acrypto_user_direct_kern *k) +{ + k->svma = kmalloc(sizeof(struct vm_area_struct *) * k->snum, GFP_KERNEL); + if (!k->svma) { + dprintk("Failed to allocate %d source vmas.\n", k->snum); + return -ENOMEM; + } + + k->dvma = kmalloc(sizeof(struct vm_area_struct *) * k->dnum, GFP_KERNEL); + if (!k->dvma) { + dprintk("Failed to allocate %d destination vmas.\n", k->dnum); + kfree(k->svma); + return -ENOMEM; + } + + return 0; +} + +static void acrypto_user_direct_free_vmas(struct acrypto_user_direct_kern *k) +{ + kfree(k->svma); + kfree(k->dvma); +} + +static int acrypto_user_direct_alloc_mm(struct acrypto_user_direct_kern *k) +{ + int err; + + err = acrypto_user_direct_alloc_pages(k); + if (err) + return err; + + err = acrypto_user_direct_alloc_vmas(k); + if (err) { + acrypto_user_direct_free_pages(k); + return err; + } + + return 0; +} + +static void acrypto_user_direct_free_mm(struct acrypto_user_direct_kern *k) +{ + acrypto_user_direct_free_pages(k); + acrypto_user_direct_free_vmas(k); +} + +static void acrypto_user_direct_free_data(struct acrypto_user_direct_kern *k) +{ + int i; + + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); + for (i=0; idnum; ++i) { + set_page_dirty_lock(k->dp[i]); + page_cache_release(k->dp[i]); + } + up_read(&k->mm->mmap_sem); + acrypto_user_direct_free_mm(k); + mmput(k->mm); +} + +static void acrypto_user_direct_callback(struct acrypto_session_initializer *ci, struct acrypto_data *data) +{ + struct acrypto_user_direct_kern *k = data->priv; + struct cn_msg m; + + memset(&m, 0, sizeof(m)); + + memcpy(&m.id, &acrypto_conn_id, sizeof(m.id)); + m.seq = k->seq; + m.ack = k->ack+1; + + cn_netlink_send(&m, 0, GFP_KERNEL); + + acrypto_user_direct_free_data(k); + acrypto_user_free_acrypto_data(data); +} + +static void acrypto_user_direct_fill_data(struct acrypto_data *data, struct acrypto_user_direct_kern *k) +{ + int i, size; + + size = k->usr.src_size; + for (i=0; isg_src_num; ++i) { + data->sg_src[i].page = k->sp[i]; + + if (i == 0) { + data->sg_src[i].offset = offset_in_page(k->usr.src); + data->sg_src[i].length = ALIGN_DATA_SIZE(k->usr.src) - k->usr.src; + } else { + data->sg_src[i].offset = 0; + data->sg_src[i].length = (i != data->sg_src_num)?PAGE_SIZE:size; + } + + size -= data->sg_src[i].length; + } + + size = k->usr.dst_size; + for (i=0; isg_dst_num; ++i) { + data->sg_dst[i].page = k->dp[i]; + + if (i == 0) { + data->sg_dst[i].offset = offset_in_page(k->usr.dst); + data->sg_dst[i].length = ALIGN_DATA_SIZE(k->usr.dst) - k->usr.dst; + } else { + data->sg_dst[i].offset = 0; + data->sg_dst[i].length = (i != data->sg_dst_num)?PAGE_SIZE:size; + } + + size -= data->sg_dst[i].length; + } +} + +static int acrypto_user_direct_prepare_data(struct acrypto_data *data, struct acrypto_user_direct_kern *k) +{ + int err, i; + struct task_struct *tsk; + + tsk = find_task_by_pid(k->usr.pid); + if (!tsk) { + dprintk(KERN_ERR "Task with pid=%d does not exist.\n", k->usr.pid); + return -ENODEV; + } + + dprintk("Found task with pid=%d.\n", k->usr.pid); + + k->mm = get_task_mm(tsk); + if (!k->mm) + return -EINVAL; + + k->snum = data->sg_src_num; + k->dnum = data->sg_dst_num; + + err = acrypto_user_direct_alloc_mm(k); + if (err) + goto err_out_put_mm; + + down_read(&k->mm->mmap_sem); + + err = get_user_pages(tsk, k->mm, k->usr.src, k->snum, 1, 1, k->sp, k->svma); + if (err < 0) { + dprintk("Failed to get %d src pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_up_sem; + } + + err = get_user_pages(tsk, k->mm, k->usr.dst, k->dnum, 1, 1, k->dp, k->dvma); + if (err < 0) { + dprintk("Failed to get %d dst pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_put_src; + } + + acrypto_user_direct_fill_data(data, k); + + return 0; + +err_out_put_src: + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); +err_out_up_sem: + up_read(&k->mm->mmap_sem); + + acrypto_user_direct_free_mm(k); +err_out_put_mm: + mmput(k->mm); + return err; +} + +static int acrypto_user_direct_prepare(struct acrypto_session_initializer *ci, struct acrypto_data *data, struct acrypto_user_direct_kern *k) +{ + int err; + + ci->operation = k->usr.operation; + ci->type = k->usr.type; + ci->mode = k->usr.mode; + ci->priority = k->usr.priority; + ci->callback = acrypto_user_direct_callback; + + err = acrypto_user_alloc_acrypto_data(data, k->usr.src_size, k->usr.dst_size, k->usr.key_size, k->usr.iv_size); + if (err) + return err; + + if (k->usr.key_size) + acrypto_user_fill_sg(k->key, k->usr.key_size, data->sg_key); + + if (k->usr.iv_size) + acrypto_user_fill_sg(k->iv, k->usr.iv_size, data->sg_iv); + + data->priv = k; + data->priv_size = 0; + + err = acrypto_user_direct_prepare_data(data, k); + if (err) { + acrypto_user_free_acrypto_data(data); + return err; + } + + return 0; +} + +static int acrypto_user_direct_process(struct acrypto_user_direct_kern *k) +{ + struct acrypto_session_initializer ci; + struct acrypto_data data; + struct acrypto_session *s; + int err; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + err = acrypto_user_direct_prepare(&ci, &data, k); + if (err) + return err; + + s = acrypto_session_alloc(&ci, &data); + if (!s) { + acrypto_user_direct_free_data(k); + return -EINVAL; + } + + return 0; +} + +static int acrypto_user_direct_thread(void *data) +{ + struct acrypto_user_direct_kern *k; + + daemonize("acrypto_user_direct_thread"); + allow_signal(SIGTERM); + + + while (!need_exit) { + wait_event_interruptible_timeout(acrypto_user_direct_wait_queue, + acrypto_user_direct_has_work, 1000); + acrypto_user_direct_has_work = 0; + + spin_lock_bh(&acrypto_user_direct_lock); + while(!list_empty(&acrypto_user_direct_list)) { + k = list_entry(acrypto_user_direct_list.prev, + struct acrypto_user_direct_kern, entry); + list_del(&k->entry); + spin_unlock_bh(&acrypto_user_direct_lock); + + acrypto_user_direct_process(k); + + spin_lock_bh(&acrypto_user_direct_lock); + } + spin_unlock_bh(&acrypto_user_direct_lock); + } + + complete_and_exit(&thread_exited, 0); +} + +int acrypto_user_direct_add_request(u32 seq, u32 ack, struct acrypto_user_direct *usr) +{ + struct acrypto_user_direct_kern *k; + + k = kmalloc(sizeof(struct acrypto_user_direct_kern) + usr->key_size + usr->iv_size, GFP_ATOMIC); + if (!k) { + dprintk("Failed to allocate new kernel acrypto request.\n"); + return -ENOMEM; + } + + memset(k, 0, sizeof(*k)); + + memcpy(&k->usr, usr, sizeof(k->usr)); + + k->key = (u8 *)(k+1); + k->iv = (u8 *)(k->key + k->usr.key_size); + + memcpy(k->key, usr->data, k->usr.key_size); + memcpy(k->iv, usr->data + k->usr.key_size, k->usr.iv_size); + + k->seq = seq; + k->ack = ack; + + spin_lock_bh(&acrypto_user_direct_lock); + list_add_tail(&k->entry, &acrypto_user_direct_list); + spin_unlock_bh(&acrypto_user_direct_lock); + + dprintk("msg [%08x.%08x]: op=[%04x.%04x.%04x.%04x], src=%llx [%d], dst=%llx [%d], key=%p [%d], iv=%p [%d].\n", + seq, ack, + k->usr.operation, k->usr.mode, k->usr.type, k->usr.priority, + k->usr.src, k->usr.src_size, + k->usr.dst, k->usr.dst_size, + k->key, k->usr.key_size, + k->iv, k->usr.iv_size); + + acrypto_user_direct_has_work = 1; + wake_up_interruptible(&acrypto_user_direct_wait_queue); + + return 0; +} + +int acrypto_user_direct_init(void) +{ + long pid; + + init_completion(&thread_exited); + pid = kernel_thread(acrypto_user_direct_thread, NULL, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + dprintk(KERN_ERR "Failed to create acrypto userspace processing thread.\n"); + return -EINVAL; + } + + printk(KERN_INFO "Aacrypto userspace processing thread has been started.\n"); + + return 0; +} + +void acrypto_user_direct_fini(void) +{ + need_exit = 1; + wait_for_completion(&thread_exited); + + printk(KERN_INFO "Aacrypto userspace processing thread has been finished.\n"); +} diff --git a/acrypto/acrypto_user_direct.h b/acrypto/acrypto_user_direct.h new file mode 100644 index 0000000..703a041 --- /dev/null +++ b/acrypto/acrypto_user_direct.h @@ -0,0 +1,74 @@ +/* + * acrypto_user_direct.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_USER_DIRECT_H +#define __ACRYPTO_USER_DIRECT_H + +struct acrypto_user_direct +{ + __u64 src; + __u32 src_size; + __u64 dst; + __u32 dst_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; + + int pid; + + int key_size; + int iv_size; + + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +#include +#include +#include + +struct acrypto_user_direct_kern +{ + struct list_head entry; + + u32 seq; + u32 ack; + + struct acrypto_user_direct usr; + u8 *key; + u8 *iv; + + int snum, dnum; + struct page **sp, **dp; + struct vm_area_struct **svma, **dvma; + struct mm_struct *mm; +}; + +int acrypto_user_direct_init(void); +void acrypto_user_direct_fini(void); +int acrypto_user_direct_add_request(u32 seq, u32 ack, struct acrypto_user_direct *usr); + +#endif /* __KERNEL__ */ + +#endif /* __ACRYPTO_USER_DIRECT_H */ diff --git a/acrypto/acrypto_user_ioctl.c b/acrypto/acrypto_user_ioctl.c new file mode 100644 index 0000000..f735885 --- /dev/null +++ b/acrypto/acrypto_user_ioctl.c @@ -0,0 +1,263 @@ +/* + * acrypto_user_ioctl.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "acrypto_user.h" +#include "acrypto_user_ioctl.h" + +static int acrypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg); +static int acrypto_user_ioctl_open(struct inode *inode, struct file *fp); +static int acrypto_user_ioctl_release(struct inode *pinode, struct file *fp); +int acrypto_user_ioctl_init(void); +void acrypto_user_ioctl_fini(void); + +static int acrypto_user_ioctl_major = 0; +static char acrypto_user_ioctl_name[] = "acrypto_user_ioctl"; + +static struct file_operations acrypto_user_ioctl_ops = { + .open = acrypto_user_ioctl_open, + .release = acrypto_user_ioctl_release, + .ioctl = acrypto_user_ioctl_ioctl, + .owner = THIS_MODULE +}; + +static void dump_data(u8 *ptr) +{ + int i; + + dprintk("USER DATA: "); + for (i=0; i<32; ++i) + dprintka("%02x ", ptr[i]); + dprintka("\n"); +} + +static int acrypto_user_ioctl_open(struct inode *inode, struct file *fp) +{ + struct acrypto_user_ioctl_kern *iok; + + iok = kmalloc(sizeof(*iok), GFP_KERNEL); + if (!iok) { + dprintk("Failed to allocate new acrypto_user_ioctl_kern structure.\n"); + return -ENOMEM; + } + memset(iok, 0, sizeof(*iok)); + + fp->private_data = iok; + + return 0; +} + +static int acrypto_user_ioctl_release(struct inode *pinode, struct file *fp) +{ + struct acrypto_user_ioctl_kern *iok = fp->private_data; + int i; + + for (i=0; i<4; ++i) + if (iok->ptr[i]) + kfree(iok->ptr[i]); + kfree(iok); + + return 0; +} + +static void acrypto_user_ioctl_callback(struct acrypto_session_initializer *ci, struct acrypto_data *data) +{ + struct acrypto_user_ioctl_kern *iok = data->priv; + + dprintk("%s() for session %08x [%08x].\n", + __func__, iok->s->ci.id, iok->s->ci.dev_id); + + acrypto_user_free_acrypto_data(&iok->data); + + iok->scompleted = 1; + wake_up_interruptible(&iok->wait); +} + +static int acrypto_user_ioctl_session_alloc(struct acrypto_user_ioctl *io, struct acrypto_user_ioctl_kern *iok) +{ + int err; + + err = acrypto_user_alloc_acrypto_data(&iok->data, io->src_size, io->dst_size, io->key_size, io->iv_size); + if (err) + return err; + + iok->ci.operation = io->operation; + iok->ci.type = io->type; + iok->ci.mode = io->mode; + iok->ci.priority = io->priority; + iok->ci.callback = acrypto_user_ioctl_callback; + + iok->data.priv = iok; + iok->data.priv_size = 0; + + iok->scompleted = 0; + + init_waitqueue_head(&iok->wait); + + iok->s = acrypto_session_create(&iok->ci, &iok->data); + if (!iok->s) { + acrypto_user_free_acrypto_data(&iok->data); + return -ENODEV; + } + + return 0; +} + +static int acrypto_user_ioctl_session_add(struct acrypto_user_ioctl_kern *iok) +{ + acrypto_session_add(iok->s); + + return 0; +} + +static int acrypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg) +{ + struct acrypto_user_ioctl io; + struct acrypto_user_data data; + unsigned long not_read; + int err; + struct acrypto_user_ioctl_kern *iok; + + iok = fp->private_data; + + err = 0; + switch (cmd) { + case ACRYPTO_SESSION_ALLOC: + not_read = copy_from_user(&io, (void __user *)arg, sizeof(io)); + if (not_read) { + dprintk("Failed to read acrypto_user_ioctl structure from userspace.\n"); + err = -EINVAL; + break; + } + + err = acrypto_user_ioctl_session_alloc(&io, iok); + break; + case ACRYPTO_FILL_DATA: + not_read = copy_from_user(&data, (void __user *)arg, sizeof(data)); + if (not_read) { + dprintk("Failed to read acrypto_user_ioctl_data structure from userspace.\n"); + err = -EINVAL; + break; + } + + if (data.data_size > MAX_DATA_SIZE * PAGE_SIZE) { + dprintk("Data size is too bit: size=%u, type=%x.\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (!acrypto_user_get_sg(&data, &iok->data)) { + dprintk("Invalid acrypto_user_data structure [size=%u, type=%x].\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (iok->ptr[data.data_type]) + kfree(iok->ptr[data.data_type]); + + iok->ptr[data.data_type] = kmalloc(data.data_size, GFP_KERNEL); + if (!iok->ptr[data.data_type]) { + dprintk("Failed to allocate %d bytes for data type %d.\n", + data.data_size, data.data_type); + err = -ENOMEM; + break; + } + + not_read = copy_from_user(iok->ptr[data.data_type], (void __user *)arg + sizeof(data), data.data_size); + if (not_read) { + dprintk("Failed to read %d bytes of acrypto data [type=%d] from userspace.\n", + data.data_size, data.data_type); + kfree(iok->ptr[data.data_type]); + err = -EINVAL; + break; + } + + memcpy(&iok->usr[data.data_type], &data, sizeof(struct acrypto_user_data)); + + err = acrypto_user_fill_sg_data(&data, &iok->data, iok->ptr[data.data_type]); + if (err) { + kfree(iok->ptr[data.data_type]); + break; + } + break; + case ACRYPTO_SESSION_ADD: + if (!iok->s) { + dprintk("ACRYPTO_SESSION_ADD must be called after session initialisation.\n"); + err = -EINVAL; + break; + } + + err = acrypto_user_ioctl_session_add(iok); + if (err) + break; + + wait_event_interruptible(iok->wait, iok->scompleted); + + dump_data(iok->ptr[ACRYPTO_USER_DATA_DST]); + + not_read = copy_to_user((void __user *)arg, iok->ptr[ACRYPTO_USER_DATA_DST], iok->usr[ACRYPTO_USER_DATA_DST].data_size); + if (not_read) { + dprintk("Failed to copy to user %d bytes of result.\n", iok->usr[ACRYPTO_USER_DATA_DST].data_size); + err = -EINVAL; + break; + } + break; + + default: + dprintk("Invalid ioctl(0x%x).\n", cmd); + err = -ENODEV; + break; + } + + return err; +} + +int acrypto_user_ioctl_init(void) +{ + acrypto_user_ioctl_major = register_chrdev(0, acrypto_user_ioctl_name, &acrypto_user_ioctl_ops); + if (acrypto_user_ioctl_major < 0) { + dprintk("Failed to register %s char device: err=%d.\n", acrypto_user_ioctl_name, acrypto_user_ioctl_major); + return -ENODEV; + }; + + printk("Asynchronous acrypto userspace helper(ioctl based) has been started, major=%d.\n", acrypto_user_ioctl_major); + + return 0; +} + +void acrypto_user_ioctl_fini(void) +{ + unregister_chrdev(acrypto_user_ioctl_major, acrypto_user_ioctl_name); +} diff --git a/acrypto/acrypto_user_ioctl.h b/acrypto/acrypto_user_ioctl.h new file mode 100644 index 0000000..4dc59dd --- /dev/null +++ b/acrypto/acrypto_user_ioctl.h @@ -0,0 +1,67 @@ +/* + * acrypto_user_ioctl.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_USER_IOCTL_H +#define __ACRYPTO_USER_IOCTL_H + +struct acrypto_user_ioctl +{ + __u16 src_size; + __u16 dst_size; + __u16 key_size; + __u16 iv_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; +}; + +#define ACRYPTO_USER_IOCTL_SYM 'U' +#define ACRYPTO_SESSION_ALLOC _IOW(ACRYPTO_USER_IOCTL_SYM, 0, struct acrypto_user_ioctl) +#define ACRYPTO_SESSION_ADD _IOR(ACRYPTO_USER_IOCTL_SYM, 1, char *) +#define ACRYPTO_FILL_DATA _IOW(ACRYPTO_USER_IOCTL_SYM, 2, struct acrypto_user_data) + + +#ifdef __KERNEL__ + +#include + +#include "acrypto_user.h" + +struct acrypto_user_ioctl_kern +{ + struct acrypto_session_initializer ci; + struct acrypto_data data; + struct acrypto_session *s; + + int scompleted; + wait_queue_head_t wait; + + struct acrypto_user_data usr[4]; + void *ptr[4]; +}; + +int acrypto_user_ioctl_init(void); +void acrypto_user_ioctl_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __ACRYPTO_USER_IOCTL_H */ diff --git a/acrypto/async_provider.c b/acrypto/async_provider.c new file mode 100644 index 0000000..80c0082 --- /dev/null +++ b/acrypto/async_provider.c @@ -0,0 +1,266 @@ +/* + * async_provider.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "acrypto_route.h" +#include "acrypto_user.h" + +static unsigned int trnum = 1; +module_param(trnum, uint, 0); + +static void prov_data_ready(struct acrypto_device *); + +static struct acrypto_capability prov_caps[] = { + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB, 1000}, + + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC, 1000}, +}; +static int prov_cap_number = sizeof(prov_caps)/sizeof(prov_caps[0]); + +static int need_exit; +static char async_algo[] = "aes"; +static char async_key[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +static struct acrypto_device pdev = { + .name = "async_provider", + .data_ready = prov_data_ready, + .cap = &prov_caps[0], +}; + +static struct async_provider +{ + int num; + struct crypto_tfm *tfm; + int work; + wait_queue_head_t async_wait_queue; + struct completion thread_exited; + struct acrypto_device pdev; + int setup; +} *aprov; + +static void prov_data_ready(struct acrypto_device *dev) +{ + struct async_provider *p; + + p = (struct async_provider *)dev->priv; + + if (p) { + p->work = 1; + wake_up_interruptible(&p->async_wait_queue); + } +} + +int async_process(struct acrypto_session *s, struct scatterlist *src, struct scatterlist *dst, int nbytes, + void *key, int klen, void *iv, int ilen, void *data) +{ + int err = -EINVAL; + struct async_provider *p = (struct async_provider *)data; + + if (!p->setup && key && klen) { + err = crypto_cipher_setkey(p->tfm, key, klen); + if (err) { + printk("Failed to set key [keylen=%d]: err=%d.\n", klen, err); + return err; + } + + if (s->ci.mode != ACRYPTO_MODE_ECB) { + if (!iv || !ilen || (ilen != crypto_tfm_alg_blocksize(p->tfm))) { + printk("Crypto mode %d requires IV, which is broken: iv=%p, ivlen=%d, blocksize=%u.\n", + s->ci.mode, iv, ilen, crypto_tfm_alg_blocksize(p->tfm)); + return -EINVAL; + } + + crypto_cipher_set_iv(p->tfm, iv, ilen); + } + p->setup = 1; + } + + if (s->ci.operation == ACRYPTO_OP_ENCRYPT) + err = crypto_cipher_encrypt(p->tfm, dst, src, nbytes); + else + err = crypto_cipher_decrypt(p->tfm, dst, src, nbytes); + + return err; +} + +static void async_finish(struct acrypto_session *s, void *key, int klen, void *iv, int ilen, void *data) +{ + struct async_provider *p = (struct async_provider *)data; + + if (s->ci.mode != ACRYPTO_MODE_ECB) + crypto_cipher_get_iv(p->tfm, iv, ilen); + + p->setup = 0; + set_bit(SESSION_DIRECT, &s->ci.flags); +} + +static int async_thread(void *data) +{ + struct async_provider *p = (struct async_provider *)data; + struct acrypto_device *dev = &p->pdev; + struct acrypto_session *s; + + daemonize("%s", dev->name); + allow_signal(SIGTERM); + + while (!need_exit) { + p->work = 0; + while ((s = acrypto_session_dequeue(dev)) != NULL) { + acrypto_process(s, NULL, &async_process, &async_finish, data); + } + + wait_event_interruptible_timeout(p->async_wait_queue, p->work, HZ); + } + + complete_and_exit(&p->thread_exited, 0); +} + +static int prov_init_one(struct async_provider *p) +{ + long pid; + int err; + + init_waitqueue_head(&p->async_wait_queue); + + p->tfm = crypto_alloc_tfm(async_algo, CRYPTO_TFM_MODE_CBC); + if (!p->tfm) { + printk(KERN_ERR "Failed to allocate %d's %s tfm.\n", p->num, async_algo); + return -EINVAL; + } + + err = crypto_cipher_setkey(p->tfm, async_key, sizeof(async_key)); + if (err) { + printk("Failed to set key [keylen=%zu]: err=%d.\n", + sizeof(async_key), err); + goto err_out_free_tfm; + } + + init_completion(&p->thread_exited); + + memcpy(&p->pdev, &pdev, sizeof(pdev)); + snprintf(p->pdev.name, sizeof(p->pdev.name), "async_provider%d", p->num); + + p->pdev.cap_number = prov_cap_number; + p->pdev.priv = p; + + pid = kernel_thread(async_thread, p, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + err = -EINVAL; + printk(KERN_ERR "Failed to create kernel load balancing thread.\n"); + goto err_out_free_tfm; + } + + err = acrypto_device_add(&p->pdev); + if (err) + goto err_out_remove_thread; + + return 0; + +err_out_remove_thread: + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); +err_out_free_tfm: + crypto_free_tfm(p->tfm); + + return err; +} + +static void prov_fini_one(struct async_provider *p) +{ + acrypto_device_remove(&p->pdev); + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); + + crypto_free_tfm(p->tfm); +} + +int prov_init(void) +{ + int err, i; + + aprov = kmalloc(trnum * sizeof(struct async_provider), GFP_KERNEL); + if (!aprov) { + printk(KERN_ERR "Failed to allocate %d async_provider pointers.\n", trnum); + return -ENOMEM; + } + + memset(aprov, 0, trnum * sizeof(struct async_provider)); + + for (i=0; i= 0) + prov_fini_one(&aprov[i]); + + kfree(aprov); + + return err; +} + +void prov_fini(void) +{ + int i; + + for (i=0; i"); +MODULE_DESCRIPTION("Test crypto module provider."); diff --git a/acrypto/consumer.c b/acrypto/consumer.c new file mode 100644 index 0000000..29c01d8 --- /dev/null +++ b/acrypto/consumer.c @@ -0,0 +1,266 @@ +/* + * consumer.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef DEBUG +#include +#include +#include "acrypto_user.h" + +#undef dprintk +//#define dprintk(f, a...) printk(f, ##a) +#define dprintk(f, a...) do {} while(0) + +#define KEY_LENGTH 16 +static char ckey[KEY_LENGTH]; +static char civ[KEY_LENGTH]; +static int key_length = sizeof(ckey); +static int iv_length = sizeof(civ); + +static void ctest_callback(struct acrypto_session_initializer *ci, + struct acrypto_data *data); + +static struct acrypto_session_initializer ci = { + .operation = ACRYPTO_OP_ENCRYPT, + .type = ACRYPTO_TYPE_AES_128, + .mode = ACRYPTO_MODE_CBC, + .priority = 4, + .callback = ctest_callback, +}; + +static struct acrypto_data cdata; + +#define CSESSION_MAX 1000 +static struct acrypto_session *s; +static atomic_t watermark; +static struct timer_list ctimer; +static char data_str[4096] = "test message qwerty"; + +static void ctest_callback(struct acrypto_session_initializer *ci, + struct acrypto_data *data) +{ + int i, off, size, ssize; + struct scatterlist *sg; + unsigned char *ptr; + + dprintk("%s: session %llu [%llu]\n", __func__, ci->id, ci->dev_id); + dprintk("src=%s, len=%d.\n", + ((char *)page_address(data->sg_src[0].page)) + data->sg_src[0].offset, + data->sg_src[0].length); +#if 1 + ssize = 32; + + sg = &data->sg_key[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("key[%d]=", size); + for (i = 0; i < size; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_src[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("src[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_dst[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("dst[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); +#endif + atomic_dec(&watermark); +} + +int ctimer_func(void *data, int size, int op) +{ + u8 *ptr; + + if (size > PAGE_SIZE) + size = PAGE_SIZE; + + ptr = kmap_atomic(cdata.sg_src[0].page, KM_USER1); + if (!ptr) + return -1; + + memcpy(ptr + cdata.sg_src[0].offset, data, size); + cdata.sg_src[0].length = size; + cdata.sg_dst[0].length = size; + + kunmap_atomic(ptr, KM_USER1); + + ci.operation = op; + s = acrypto_session_alloc(&ci, &cdata); + if (s) + atomic_inc(&watermark); + + return (s) ? 0 : -EINVAL; +} + +static int alloc_sg(struct scatterlist *sg, void *data, int size) +{ + sg->offset = 0; + sg->page = alloc_pages(GFP_KERNEL, get_order(size)); + if (!sg->page) { + printk(KERN_ERR "Failed to allocate page.\n"); + return -ENOMEM; + } + + if (data) { + void *ptr = kmap(sg->page); + + if (!ptr) { + __free_pages(sg->page, get_order(size)); + return -ENOMEM; + } + + memcpy(ptr, data, size); + kunmap(ptr); + } + + sg->length = size; + + return 0; +} + +static void ctimerf(unsigned long data) +{ + int size, err, i; + + dprintk("%s started\n", __func__); + + size = 4096; + + for (i=0; i<1000; ++i) + err = ctimer_func(data_str, size, ACRYPTO_OP_ENCRYPT); + //err = ctimer_func(data_str, size, ACRYPTO_OP_DECRYPT); + + if (!err) + mod_timer(&ctimer, jiffies + 1); + else + mod_timer(&ctimer, jiffies + 1); + + dprintk("%s finished.\n", __func__); +} + +int consumer_init(void) +{ + int err; + + err = acrypto_user_alloc_acrypto_data(&cdata, sizeof(data_str), sizeof(data_str), key_length, iv_length); + if (err) + return err; + + err = alloc_sg(&cdata.sg_src[0], NULL, PAGE_SIZE); + if (err) + goto err_out_return; + + err = alloc_sg(&cdata.sg_dst[0], NULL, PAGE_SIZE); + if (err) + goto err_out_src; + + err = alloc_sg(&cdata.sg_key[0], ckey, key_length); + if (err) + goto err_out_dst; + + err = alloc_sg(&cdata.sg_iv[0], civ, iv_length); + if (err) + goto err_out_key; + + //cdata.priv_size = 256;//sizeof(struct aes_ctx); + + init_timer(&ctimer); + ctimer.function = &ctimerf; + ctimer.expires = jiffies + HZ; + ctimer.data = 0; + + add_timer(&ctimer); + + return 0; + + __free_pages(cdata.sg_iv[0].page, get_order(1)); +err_out_key: + __free_pages(cdata.sg_key[0].page, get_order(key_length)); +err_out_dst: + __free_pages(cdata.sg_dst[0].page, get_order(1)); +err_out_src: + __free_pages(cdata.sg_src[0].page, get_order(1)); +err_out_return: + acrypto_user_free_acrypto_data(&cdata); + + return -ENODEV; +} + +void consumer_fini(void) +{ + del_timer_sync(&ctimer); + + printk(KERN_INFO "%s: Timer has been removed.\n", __func__); + while (atomic_read(&watermark)) { + msleep(1000); + + printk(KERN_INFO "Waiting for sessions to be freed: watermark=%d.\n", + atomic_read(&watermark)); + } + + if (!cdata.sg_key[0].length) + printk("BUG: key length is 0 in %s.\n", __func__); + + __free_pages(cdata.sg_iv[0].page, get_order(1)); + __free_pages(cdata.sg_key[0].page, get_order(key_length)); + __free_pages(cdata.sg_dst[0].page, get_order(1)); + __free_pages(cdata.sg_src[0].page, get_order(1)); + + acrypto_user_free_acrypto_data(&cdata); + + printk(KERN_INFO "Test acrypto module consumer is unloaded.\n"); +} + +module_init(consumer_init); +module_exit(consumer_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Test acrypto module consumer."); diff --git a/acrypto/ocf_acrypto.c b/acrypto/ocf_acrypto.c new file mode 100644 index 0000000..bc96e33 --- /dev/null +++ b/acrypto/ocf_acrypto.c @@ -0,0 +1,852 @@ +/* + * ocf_acrypto.c - Acrypto to OCF bridge driver. + * + * Yakov Lerner + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "acrypto_route.h" +#include "acrypto_user.h" +#include "../crypto/ocf/cryptodev.h" // ocf + +// 2006-07-07 lerner multifragment handling completed. multifragment works +// properly on top of ixp4xx driver (which itself dosn't handle +// multifragment requests). + +#undef dprintk +#if 0 +#define dprintk(a...) printk(a) +#define DEBUG_IV +#define DEBUG +#else +#define dprintk(a...) +#undef DEBUG +#endif + + +static const char modname[] = "acrypto2ocf"; + +#ifndef AES_BLOCK_SIZE +#define AES_BLOCK_SIZE 16 +#endif +#ifndef DES3_BLOCK_SIZE +#define DES3_BLOCK_SIZE 8 +#endif +#define MAX_BLOCKSIZE 16 + +static spinlock_t ctxlist_lock = SPIN_LOCK_UNLOCKED; +static struct list_head ctxlist = LIST_HEAD_INIT(ctxlist); + +static int nocopy = 0; +module_param(nocopy, int, 0); +MODULE_PARM_DESC(nocopy, "Whether to copy (linearize) skb in multifragment case"); + +static int nokmap = 1; +module_param(nokmap, int, 0); +MODULE_PARM_DESC(nokmap, "Whether to use kmap when linearizing multifragment skb"); + +// About multifragment requests. +// This driver correctly handles multifragment requests. +// This is tricky because we want this driver to work on top of ixp4xx +// OCF driver which, as of 2006-07-07, does not itself handle multifragment +// requests. +// This driver has two options for handling multigragment requests +// (1) linearize data, encrypt/decrypt, copy back. This works and +// according to banchmark, does not incur much overhead +// (2) encrypt/decrypt fragments "in-place", creating additional small "inter-fragments" +// in cases when fragment size is not dividable by blocksize. +// This works in theory and for soft-ocf-driver, but this +// does not work for ixp4xx driver. The reason is that ixp4xx software +// does not seem to returns us the updated IV after the operation. +// Our query to Intel is pending about that. + +DECLARE_WAIT_QUEUE_HEAD(waitq); /* bridge_fini waits on this until */ + /* there are no active requests */ + +int ocfbridge_context_callback( + struct acrypto_device *dev, + struct acrypto_context *ctx, + u32 flags); +static void bridge_data_ready(struct acrypto_device *dev); +static int ocf_callback(struct cryptop *crp); + + +static struct acrypto_capability bridge_caps[] = { + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB, 1000}, + + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC, 1000}, + + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_256, ACRYPTO_MODE_ECB, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_256, ACRYPTO_MODE_ECB, 1000}, + + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_256, ACRYPTO_MODE_CBC, 1000}, + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_256, ACRYPTO_MODE_CBC, 1000}, + + + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC, 1000}, + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC, 1000}, + + {ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB, 1000}, + {ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB, 1000}, + + // about BLOWFISH: OCF doesn't have define for BLOWFISH, + // as of 2006-07-03 + + // about DES: Acrypt doesn't have define for DES, + // as of 2006-07-03 + +}; +#define BRIDGE_CAP_NUMBER sizeof(bridge_caps)/sizeof(bridge_caps[0]) + + +static struct acrypto_device pdev = { + .name = "ocf_acrypto", + .data_ready = bridge_data_ready, + .cap = &bridge_caps[0], + .cap_number = BRIDGE_CAP_NUMBER, + .context_callback = ocfbridge_context_callback, + .flags = ACRYPTO_DEV_CTX_INIT | ACRYPTO_DEV_CTX_EXIT | ACRYPTO_DEV_CTX_KEY +}; + +struct priv_context { + uint64_t ocf_sid; + int key_len; + struct list_head list; /* active contexts */ + void** parent; /* &DRIVER_PRIV() */ + int blocksize; + int ocf_alg; + u8 key[]; +}; + +struct request_priv { + int sg_index; + struct acrypto_session *s; + struct priv_context *priv; + int subfrag_tail; // count of bytes not crypted at end of + // prev fragmebt, due to blocksize-rounding + uint subfrag_off; + uint subfrag_size; // bytecount of the subfragment after + // accounting for subfrag_off and subfrag_tail + uint is_interfrag; + u8 interfrag[MAX_BLOCKSIZE]; + u8 *linear_copy; + uint linear_size; +}; + + +static unsigned int completion = 0; /* 0-without queue_work; other=>with queue_work() */ +module_param(completion, uint, 0); + + +static int cipher_blocksize(u16 acrypto_type) +{ + switch( acrypto_type ) + { + case ACRYPTO_TYPE_AES_128: + case ACRYPTO_TYPE_AES_192: + case ACRYPTO_TYPE_AES_256: + return AES_BLOCK_SIZE; + case ACRYPTO_TYPE_3DES: + return DES3_BLOCK_SIZE; + default: + BUG_ON(1); + return 0; + } +} + +static int acrypto_alg_to_ocf_alg(u16 acrypto_type, u16 acrypto_mode) +{ + if( acrypto_mode != ACRYPTO_MODE_CBC) { + BUG_ON(1); + return -1; + } + + // OCF CRYPTO_AES_CBC + // OCF CRYPTO_3DES_CBC + + switch( acrypto_type ) + { + case ACRYPTO_TYPE_AES_128: + case ACRYPTO_TYPE_AES_192: + case ACRYPTO_TYPE_AES_256: + return CRYPTO_AES_CBC; /* OCF's value */ + case ACRYPTO_TYPE_3DES: + return CRYPTO_3DES_CBC; /* OCFss value */ + default: + BUG_ON(1); + return -1; + } +} + +static char *cipher_name(u16 acrypto_type, char *buf, int max) +{ + + switch( acrypto_type ) + { + case ACRYPTO_TYPE_AES_128: + strlcpy(buf, "aes128", max); + return buf; + case ACRYPTO_TYPE_AES_192: + strlcpy(buf, "aes192", max); + return buf; + case ACRYPTO_TYPE_AES_256: + strlcpy(buf, "aes256", max); + return buf; + case ACRYPTO_TYPE_3DES: + strlcpy(buf, "3des", max); + return buf; + default: + snprintf(buf, max, "?%d?", acrypto_type); + return buf; + } +} + +static char *mode_name(u16 acrypto_mode, char *buf, int max) +{ + switch (acrypto_mode) { + case ACRYPTO_MODE_CBC: + strlcpy(buf, "CBC", max); + return buf; + default: + snprintf(buf, max, "?%d?", acrypto_mode); + return buf; + } +} + +#ifdef DEBUG_IV +void print_hex(char *prefix, void *buf, int len) +{ + int k; + + printk("%s [%d] ", prefix, len); + for(k=0; kci.flags); + acrypto_complete_session(s); +} + + +static int prepare_frag_copying(struct acrypto_session *s, + struct request_priv *rq_priv, + struct priv_context *priv, + u8 **pdata, int *plength) +{ + // int is_encrypt = (s->ci.operation == ACRYPTO_OP_ENCRYPT); + u8 *frag_ptr; + uint frag_length, k; + + rq_priv->linear_copy = NULL; + dprintk("copying-dispatch\n"); + if( s->data.sg_src_num == 1) { + frag_length = s->data.sg_src[rq_priv->sg_index].length; + frag_ptr = page_address( s->data.sg_src[rq_priv->sg_index].page) + + s->data.sg_src[rq_priv->sg_index].offset; + dprintk("** dispatching single frag, size %d\n", frag_length); + } else { + u8 *linear, *kmapped; + + rq_priv->linear_size = 0; + for( k = 0; k < s->data.sg_src_num; k++) + rq_priv->linear_size += s->data.sg_src[k].length; + + rq_priv->linear_copy = kmalloc( rq_priv->linear_size, GFP_ATOMIC ); + if( rq_priv->linear_copy == NULL ) { + printk("Allocation error in %s\n", modname ); + return -ENOMEM; + } + linear = rq_priv->linear_copy; + for( k = 0; k < s->data.sg_src_num; k++) { + if (nokmap) { + frag_ptr = page_address( s->data.sg_src[k].page) + + s->data.sg_src[k].offset; + memcpy( linear, frag_ptr, s->data.sg_src[k].length); + } else { + kmapped = kmap_atomic(s->data.sg_src[k].page, KM_USER0); + if( kmapped == NULL ) { + printk("kmap error in %s\n", modname ); + goto cleanup; + } + memcpy( linear, kmapped + s->data.sg_src[k].offset, + s->data.sg_src[k].length); + kunmap_atomic( kmapped, KM_USER0); + } + linear += s->data.sg_src[k].length; + } + + frag_length = rq_priv->linear_size; + frag_ptr = rq_priv->linear_copy; + + dprintk("** dispatching linear buf, size %d, total of %d frags\n", + frag_length, s->data.sg_src_num ); + } + + *pdata = frag_ptr; + *plength = frag_length; + + return 0; +cleanup: + if( rq_priv->linear_copy ) + kfree( rq_priv->linear_copy ); + return -ENOMEM; +} + + + +static int dispatch_one_fragment( struct acrypto_session *s, + int sg_index, + struct request_priv *rq_priv, + struct priv_context *priv ) +{ + struct cryptop *crp = NULL; + struct cryptodesc *crde; + int is_encrypt = (s->ci.operation == ACRYPTO_OP_ENCRYPT); + u8 *frag_ptr; + int frag_length; + + if (nocopy == 0) { + if( prepare_frag_copying(s, rq_priv, priv, &frag_ptr, &frag_length) != 0) + return -ENOMEM; + } else { + /* Some OCF drivers, notably ixp4xx driver, does not support + * multifragment data. We deal with multifragments here and we + * turn to OCF driver with single-fragment requests only. + * There is complication, however, when fragment size + * is not divisible by algorithm blocksize. + * + * We deal with it by dividing such mis-sized fragments into + * two (a) rounded down "subfragment" of blocksize-even size, and + * (b) interfragment of exactly one-blocksize-size. + */ + + if( rq_priv->is_interfrag ) { + // copy back the interfragment data + u8 *prev_data, *prev_tail, *this_data; + int piece1 = rq_priv->subfrag_tail; + + + prev_data = page_address( s->data.sg_src[rq_priv->sg_index-1].page) + + s->data.sg_src[rq_priv->sg_index-1].offset; + prev_tail = prev_data + rq_priv->subfrag_off + rq_priv->subfrag_size; + this_data = page_address( s->data.sg_src[rq_priv->sg_index].page) + + s->data.sg_src[rq_priv->sg_index].offset; + memcpy( prev_tail, rq_priv->interfrag, piece1); + memcpy( this_data, rq_priv->interfrag+piece1, + priv->blocksize - piece1); + } + + /* decide whether to dispatch full fragment, subfragment, or */ + /* interfragment */ + if( rq_priv->subfrag_tail && ! rq_priv->is_interfrag && + rq_priv->sg_index < s->data.sg_src_num && + rq_priv->sg_index > 0 ) + { + u8 *prev_data, *this_data; + int piece1 = rq_priv->subfrag_tail; + + + dprintk("** interfragnment (%d+%d) between frag %d and %d of %d\n", + piece1, priv->blocksize - piece1, rq_priv->sg_index-1, + rq_priv->sg_index, s->data.sg_src_num ); + // rq_priv->is_interfrag = 1; + + + prev_data = page_address( s->data.sg_src[rq_priv->sg_index-1].page) + + s->data.sg_src[rq_priv->sg_index-1].offset + + rq_priv->subfrag_off + rq_priv->subfrag_size; + BUG_ON( rq_priv->subfrag_tail >= priv->blocksize ); + memcpy(rq_priv->interfrag, prev_data, rq_priv->subfrag_tail ); + + this_data = page_address( s->data.sg_src[rq_priv->sg_index].page) + + s->data.sg_src[rq_priv->sg_index].offset; + BUG_ON( s->data.sg_src[rq_priv->sg_index].length < priv->blocksize - piece1); + memcpy(rq_priv->interfrag + piece1, this_data, priv->blocksize - piece1); + BUG_ON(sizeof(rq_priv->interfrag) < priv->blocksize); + + frag_length = priv->blocksize; + frag_ptr = rq_priv->interfrag; + + // rq_priv->subfrag_size = rq_priv->subfrag_off = 0; + rq_priv->is_interfrag = 1; + + rq_priv->sg_index --; + } else { + // this is going to be full fragment or subfragment. + + + rq_priv->is_interfrag = 0; + + rq_priv->subfrag_off = ( rq_priv->subfrag_tail == 0 ? 0 : + priv->blocksize - rq_priv->subfrag_tail); + // prev tail affects current subfrag_off + + rq_priv->subfrag_size = s->data.sg_src[rq_priv->sg_index].length - + rq_priv->subfrag_off; + rq_priv->subfrag_tail = rq_priv->subfrag_size % priv->blocksize; + rq_priv->subfrag_size -= rq_priv->subfrag_tail; + + frag_length = rq_priv->subfrag_size; + frag_ptr = page_address( s->data.sg_src[rq_priv->sg_index].page) + + s->data.sg_src[rq_priv->sg_index].offset + + rq_priv->subfrag_off; + dprintk("** %s fragment No. %d of %d; size=%d, off=%d, tail=%d\n", + (rq_priv->subfrag_off==0 && rq_priv->subfrag_tail==0 ? "full" : + "partial"), rq_priv->sg_index, s->data.sg_src_num, + rq_priv->subfrag_size, rq_priv->subfrag_off, + rq_priv->subfrag_tail ); + } + } + + + crp = crypto_getreq(1); + if (!crp) + { + printk(KERN_NOTICE "crypto_getreq failed\n"); + return -ENOMEM; + } + crde = crp->crp_desc; + + dprintk("** dispatching frag #%d of %d\n", rq_priv->sg_index, s->data.sg_src_num ); + crde->crd_skip = 0; + if (is_encrypt) + crde->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_ENCRYPT | CRD_F_IV_PRESENT; + else + crde->crd_flags = CRD_F_IV_EXPLICIT; + // CRD_F_IV_PRESENT is ignored for decrypt. + crde->crd_len = frag_length; + crde->crd_inject = 0; + crde->crd_alg = priv->ocf_alg; // CRYPTO_AES_CBC, CRYPTO_3DES_CBC + + if (s->data.sg_iv_num) { + u8 *iv; + int ivlen; + + iv = page_address( s->data.sg_iv[0].page) + s->data.sg_iv[0].offset; + ivlen = s->data.sg_iv[0].length; + memcpy(crde->crd_iv, iv, min(EALG_MAX_BLOCK_LEN, ivlen)); +#ifdef DEBUG_IV + print_hex((is_encrypt?"TX iv before":"RX iv before"), crde->crd_iv, ivlen ); +#endif + } + + crp->crp_ilen = frag_length; + crp->crp_flags = 0; /* CRYPTO_F_CBIMM; */ + crp->crp_buf = frag_ptr; + crp->crp_callback = ocf_callback; + crp->crp_sid = priv->ocf_sid; + rq_priv->s = s; + rq_priv->priv = priv; + crp->crp_opaque = (caddr_t)rq_priv; + + //crypto_dispatch(crp); + crypto_invoke(crp, 0); // crypto_invoke() is like crypto_dispatch_direct() + + return 0; +} + + +static int +ocf_callback(struct cryptop *crp) +{ + struct request_priv *rq_priv = (struct request_priv*)crp->crp_opaque; + struct acrypto_session *s = rq_priv->s; + //struct cryptodesc *crde = crp->crp->crp_desc; + //int is_encrypt = (s->ci.operation == ACRYPTO_OP_ENCRYPT); + + dprintk("**** ocf cb entry s=%p, index=%d, nfrag=%d\n", + s, rq_priv->sg_index, s->data.sg_src_num); + + //if (s->ci.operation == ACRYPTO_OP_ENCRYPT) + { + u8 *iv; + int ivlen; + + iv = page_address(s->data.sg_iv[0].page) + s->data.sg_iv[0].offset; + ivlen = s->data.sg_iv[0].length; + +#ifdef DEBUG_IV + print_hex(((s->ci.operation == ACRYPTO_OP_ENCRYPT)?"TX iv after": + "RX iv after"), + crp->crp_desc->crd_iv, ivlen ); +#endif + memcpy( iv, crp->crp_desc->crd_iv, ivlen ); + } + + crypto_freereq( crp ); /* destroy OCF request */ + crp = NULL; + + if( nocopy == 0) { + if( rq_priv->linear_copy ) { + u8 *linear, *kmapped, *p; + int k; + + dprintk("copying linearized data back\n"); + linear = rq_priv->linear_copy; + for( k = 0; k < s->data.sg_src_num; k++) { + if (nokmap) { + p = page_address( s->data.sg_src[k].page) + + s->data.sg_src[k].offset; + memcpy( p, linear, s->data.sg_src[k].length); + } else { + kmapped = kmap_atomic(s->data.sg_src[k].page, KM_USER0); + if( kmapped == NULL ) { + printk("kmap error in %s\n", modname ); + } else { + memcpy( kmapped + s->data.sg_src[k].offset, linear, + s->data.sg_src[k].length); + } + kunmap_atomic( kmapped, KM_USER0); + } + linear += s->data.sg_src[k].length; + } + + kfree( rq_priv->linear_copy ); + } +#ifdef DEBUG + { + u8 *data = page_address(s->data.sg_src[0].page) + s->data.sg_src[0].offset; + print_hex(((s->ci.operation == ACRYPTO_OP_ENCRYPT)?"TX data": + "RX data"), data, 20); + } +#endif + acrypto_complete_session( rq_priv->s ); + + goto cleanup_after_request; + } else { + + rq_priv->sg_index ++; + + if( rq_priv->sg_index < s->data.sg_src_num ) { + dprintk("**** ocf cb next frag\n"); + /* process next sg data element */ + + if (dispatch_one_fragment( s, rq_priv->sg_index, rq_priv, rq_priv->priv )) + { + bounce_request(s, 0); /* we are in callback, bounce is asynchronous */ + goto cleanup_after_request; + } + + } else { + dprintk("**** ocf cb final frag done\n"); + /* request completed */ + + acrypto_complete_session( rq_priv->s ); + + goto cleanup_after_request; + } + } + + return 0; + +cleanup_after_request: + if (crp) + crypto_freereq( crp ); + + kfree( rq_priv ); + + //atomic_dec( &active_requests ); + //wake_up( &waitq ); + + return 0; +} + + +static void initiate_request(struct acrypto_session *s) +{ + struct priv_context *priv; + struct request_priv *rq_priv = NULL; +#ifdef DEBUG + static char tmp1[20], tmp2[20]; +#endif + + if( s->data.ctx == NULL ) { + printk(KERN_NOTICE "Request has NULL context in %s\n", modname ); + bounce_request(s, 1); + return; + } + priv = DRIVER_PRIV(s->pool_dev, s->data.ctx); + +#ifdef DEBUG + dprintk("ocf initiate_request, type=%s, mode=%s, s=%p, keylen=%d\n", + cipher_name(s->data.ctx->type,tmp1, sizeof(tmp1)), + mode_name(s->data.ctx->mode, tmp2, sizeof(tmp2)), s, priv->key_len); +#endif + //if (exiting) + // goto fail; + + + if (priv == NULL) { + printk(KERN_NOTICE "NULL priv context in ocf_bridge\n"); + goto fail; + } + + rq_priv = kzalloc(sizeof(*rq_priv), GFP_ATOMIC ); + if (rq_priv == NULL) { + printk(KERN_NOTICE "alloc failure in %s\n", modname ); + goto fail; + } + + //atomic_inc( &active_requests ); + + if (dispatch_one_fragment( s, 0, rq_priv, priv)) { + //atomic_dec( &active_requests ); + goto fail; + } + + return; + +fail: + if (rq_priv) + kfree( rq_priv ); + bounce_request(s, 1); +} + + +static void bridge_data_ready(struct acrypto_device *dev) +{ + struct acrypto_session *s; + + while ((s = acrypto_session_dequeue(dev)) != NULL) { + initiate_request(s); + } +} + + +void free_context( struct priv_context *priv, int take_lock ) +{ + dprintk("entry to free_context\n"); + *(priv->parent) = NULL; + + if( take_lock ) + spin_lock( &ctxlist_lock ); + list_del( &priv->list ); + if( take_lock ) + spin_unlock( &ctxlist_lock ); + + crypto_freesession( priv->ocf_sid ); + priv->ocf_sid = -1; + + kfree( priv ); + dprintk("return from free_context\n"); +} + + + +int ocfbridge_context_callback( + struct acrypto_device *dev, + struct acrypto_context *ctx, + u32 flags) +{ +// ACRYPTO_DEV_CTX_INIT +// ACRYPTO_DEV_CTX_EXIT +// ACRYPTO_DEV_CTX_KEY + + int err; + struct priv_context *priv; + + dprintk("ocfbridge_context_callback entry 110\n"); + switch (flags) + { + + case ACRYPTO_DEV_CTX_INIT: + { + struct cryptoini crie; + uint64_t sid; + int ocf_alg; + +create_context: + printk(KERN_NOTICE "CTX_INIT request in %s\n", modname ); + priv = DRIVER_PRIV(dev,ctx); + if (priv) { + free_context( priv, 1); + } + + ocf_alg = acrypto_alg_to_ocf_alg(ctx->type, ctx->mode); + if( ocf_alg == -1 ) { + printk( KERN_NOTICE "Bad alg (%d,%d) in %s\n", + ctx->type, ctx->mode, modname ); + return -EINVAL; + } + + memset( &crie, 0, sizeof(crie)); + crie.cri_alg = ocf_alg; // CRYPTO_AES_CBC, CRYPTO_3DES_CBC + crie.cri_klen = ctx->key_size * 8; + crie.cri_key = ctx->key; + + err = crypto_newsession( &sid, &crie, 0/*hw & sw*/); + if (err) + { + printk( KERN_NOTICE "Error creating OCF session (%d)\n", + err); + return -EFAULT; + } + + priv = kzalloc( sizeof(struct priv_context) + ctx->key_size, + GFP_KERNEL ); + DRIVER_PRIV(dev,ctx) = priv; + if (priv == NULL) { + crypto_freesession( sid ); + printk(KERN_NOTICE "Alloc error in %s\n", modname ); + return -EFAULT; + } + + priv->ocf_alg = ocf_alg; + priv->ocf_sid = sid; + priv->key_len = ctx->key_size; + priv->parent = &(DRIVER_PRIV(dev,ctx)); + priv->blocksize = cipher_blocksize( ctx->type ); + dprintk("** context blocksize=%d\n", priv->blocksize ); + memcpy(priv->key, ctx->key, priv->key_len); + printk(KERN_NOTICE "CTX_INIT done\n"); + spin_lock( &ctxlist_lock ); + list_add( &priv->list, &ctxlist ); + spin_unlock( &ctxlist_lock ); + } + break; + + case ACRYPTO_DEV_CTX_EXIT: + + printk(KERN_NOTICE "CTX_EXIT in %s\n", modname ); + priv = DRIVER_PRIV(dev,ctx); + if (priv) { + free_context( priv, 1); + } + break; + + case ACRYPTO_DEV_CTX_KEY: + // if key indeed changed, thn we must re-create the ocf session + // if key did not really change, we can just ignore the notification + + priv = DRIVER_PRIV(dev,ctx); + if (priv == NULL) + { + printk(KERN_NOTICE "%s: key-change on null context\n", modname); + BUG_ON(1); + goto create_context; + + } else if (ctx->key_size == priv->key_len || + 0 == memcmp( ctx->key, priv->key, priv->key_len )) + { + /* key did not change, notification is a no-op */ + goto ret; + + } else { + + // key changed, re-create the context + + goto create_context; + } + + break; + + default: + printk(KERN_NOTICE "Unknown flag in ocfbridge_context_callback, %x\n", + flags); + } +ret: + dprintk("return from ocfbridge_context_callback\n"); + return 0; +} + + + +int bridge_init(void) +{ + int err; + + pdev.priv = NULL; // private pointer + dprintk("&pdev=%p\n flagsd=%lx\n", &pdev, pdev.flags ); + err = acrypto_device_add( &pdev ); + if (err) { + printk(KERN_NOTICE "ERROR in acrypto_device_add; module %s\n", modname ); + return err; + } + printk(KERN_NOTICE "Module <%s> is loaded, nokmap=%d, nocopy=%d\n", + modname, nokmap, nocopy ); + + return err; +} + + + +void bridge_fini(void) +{ + struct priv_context *priv, *tmp; + +// exiting = 1; // prevent new requests to enter the service +// +// wait until all active requests are finished +// while (atomic_read(&active_requests) != 0) { +// if (-ERESTARTSYS == wait_event_interruptible( waitq, +// atomic_read(&active_requests) == 0)) +// break; +// } + + + spin_lock( &ctxlist_lock ); + list_for_each_entry_safe(priv, tmp, &ctxlist, list) { + free_context( priv, 0 ); + } + spin_unlock( &ctxlist_lock ); + + printk(KERN_NOTICE "Module <%s> is unloaded\n", modname ); +} + +/* before 20