diff --git a/Makefile b/Makefile index cb57905..305e94d 100644 --- a/Makefile +++ b/Makefile @@ -556,7 +556,7 @@ export MODLIB ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ acrypto/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/acrypto/Kconfig b/acrypto/Kconfig new file mode 100644 index 0000000..42946f1 --- /dev/null +++ b/acrypto/Kconfig @@ -0,0 +1,29 @@ +menu "Asynchronous crypto layer" + +config ACRYPTO + tristate "Asynchronous crypto layer" + select CONNECTOR + --- help --- + It supports: + - multiple asynchronous crypto device queues + - crypto session routing + - crypto session binding + - modular load balancing + - crypto session batching genetically implemented by design + - crypto session priority + - different kinds of crypto operation(RNG, asymmetrical crypto, HMAC and any other + +config ASYNC_PROVIDER + tristate "Asynchronous crypto provider (AES CBC)" + depends on ACRYPTO && (CRYPTO_AES || CRYPTO_AES_586 || CRYPTO_AES_X86_64) + --- help --- + Asynchronous crypto provider based on synchronous crypto layer. + It supports AES CBC crypto mode (may be changed by source edition). + +config CONSUMER + tristate "Test asynchronous crypto consumer" + depends on ACRYPTO + --- help --- + Test asynchronous crypto consumer. + +endmenu diff --git a/acrypto/Makefile b/acrypto/Makefile new file mode 100644 index 0000000..2c6b067 --- /dev/null +++ b/acrypto/Makefile @@ -0,0 +1,14 @@ +obj-$(CONFIG_ACRYPTO) += acrypto.o +obj-$(CONFIG_ASYNC_PROVIDER) += async_provider.o +obj-$(CONFIG_CONSUMER) += consumer.o + +acrypto-y += crypto_main.o +acrypto-y += crypto_lb.o +acrypto-y += crypto_dev.o +acrypto-y += crypto_conn.o +acrypto-y += crypto_stat.o +acrypto-y += crypto_user_direct.o +acrypto-y += crypto_user_ioctl.o +acrypto-y += crypto_user.o +acrypto-y += simple_lb.o +acrypto-y += crypto_queue.o diff --git a/acrypto/async_provider.c b/acrypto/async_provider.c new file mode 100644 index 0000000..21e3c9c --- /dev/null +++ b/acrypto/async_provider.c @@ -0,0 +1,266 @@ +/* + * async_provider.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "crypto_route.h" +#include "crypto_user.h" + +static unsigned int trnum = 1; +module_param(trnum, uint, 0); + +static void prov_data_ready(struct crypto_device *); + +static struct crypto_capability prov_caps[] = { + {CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 1000}, + {CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 1000}, + + {CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 1000}, + {CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 1000}, +}; +static int prov_cap_number = sizeof(prov_caps)/sizeof(prov_caps[0]); + +static int need_exit; +static char async_algo[] = "aes"; +static char async_key[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +static struct crypto_device pdev = { + .name = "async_provider", + .data_ready = prov_data_ready, + .cap = &prov_caps[0], +}; + +static struct async_provider +{ + int num; + struct crypto_tfm *tfm; + int work; + wait_queue_head_t async_wait_queue; + struct completion thread_exited; + struct crypto_device pdev; + int setup; +} *aprov; + +static void prov_data_ready(struct crypto_device *dev) +{ + struct async_provider *p; + + p = (struct async_provider *)dev->priv; + + if (p) { + p->work = 1; + wake_up_interruptible(&p->async_wait_queue); + } +} + +int async_process(struct crypto_session *s, struct scatterlist *src, struct scatterlist *dst, int nbytes, + void *key, int klen, void *iv, int ilen, void *data) +{ + int err = -EINVAL; + struct async_provider *p = (struct async_provider *)data; + + if (!p->setup && key && klen) { + err = crypto_cipher_setkey(p->tfm, key, klen); + if (err) { + printk("Failed to set key [keylen=%d]: err=%d.\n", klen, err); + return err; + } + + if (s->ci.mode != CRYPTO_MODE_ECB) { + if (!iv || !ilen || (ilen != crypto_tfm_alg_blocksize(p->tfm))) { + printk("Crypto mode %d requires IV, which is broken: iv=%p, ivlen=%d, blocksize=%u.\n", + s->ci.mode, iv, ilen, crypto_tfm_alg_blocksize(p->tfm)); + return -EINVAL; + } + + crypto_cipher_set_iv(p->tfm, iv, ilen); + } + p->setup = 1; + } + + if (s->ci.operation == CRYPTO_OP_ENCRYPT) + err = crypto_cipher_encrypt(p->tfm, dst, src, nbytes); + else + err = crypto_cipher_decrypt(p->tfm, dst, src, nbytes); + + return err; +} + +static void async_finish(struct crypto_session *s, void *key, int klen, void *iv, int ilen, void *data) +{ + struct async_provider *p = (struct async_provider *)data; + + if (s->ci.mode != CRYPTO_MODE_ECB) + crypto_cipher_get_iv(p->tfm, iv, ilen); + + p->setup = 0; + set_bit(SESSION_DIRECT, &s->ci.flags); +} + +static int async_thread(void *data) +{ + struct async_provider *p = (struct async_provider *)data; + struct crypto_device *dev = &p->pdev; + struct crypto_session *s; + + daemonize("%s", dev->name); + allow_signal(SIGTERM); + + while (!need_exit) { + p->work = 0; + while ((s = crypto_session_dequeue(dev)) != NULL) { + crypto_process(s, NULL, &async_process, &async_finish, data); + } + + wait_event_interruptible_timeout(p->async_wait_queue, p->work, HZ); + } + + complete_and_exit(&p->thread_exited, 0); +} + +static int prov_init_one(struct async_provider *p) +{ + long pid; + int err; + + init_waitqueue_head(&p->async_wait_queue); + + p->tfm = crypto_alloc_tfm(async_algo, CRYPTO_TFM_MODE_CBC); + if (!p->tfm) { + printk(KERN_ERR "Failed to allocate %d's %s tfm.\n", p->num, async_algo); + return -EINVAL; + } + + err = crypto_cipher_setkey(p->tfm, async_key, sizeof(async_key)); + if (err) { + printk("Failed to set key [keylen=%zu]: err=%d.\n", + sizeof(async_key), err); + goto err_out_free_tfm; + } + + init_completion(&p->thread_exited); + + memcpy(&p->pdev, &pdev, sizeof(pdev)); + snprintf(p->pdev.name, sizeof(p->pdev.name), "async_provider%d", p->num); + + p->pdev.cap_number = prov_cap_number; + p->pdev.priv = p; + + pid = kernel_thread(async_thread, p, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + err = -EINVAL; + printk(KERN_ERR "Failed to create kernel load balancing thread.\n"); + goto err_out_free_tfm; + } + + err = crypto_device_add(&p->pdev); + if (err) + goto err_out_remove_thread; + + return 0; + +err_out_remove_thread: + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); +err_out_free_tfm: + crypto_free_tfm(p->tfm); + + return err; +} + +static void prov_fini_one(struct async_provider *p) +{ + crypto_device_remove(&p->pdev); + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); + + crypto_free_tfm(p->tfm); +} + +int prov_init(void) +{ + int err, i; + + aprov = kmalloc(trnum * sizeof(struct async_provider), GFP_KERNEL); + if (!aprov) { + printk(KERN_ERR "Failed to allocate %d async_provider pointers.\n", trnum); + return -ENOMEM; + } + + memset(aprov, 0, trnum * sizeof(struct async_provider)); + + for (i=0; i= 0) + prov_fini_one(&aprov[i]); + + kfree(aprov); + + return err; +} + +void prov_fini(void) +{ + int i; + + for (i=0; i"); +MODULE_DESCRIPTION("Test crypto module provider."); diff --git a/acrypto/consumer.c b/acrypto/consumer.c new file mode 100644 index 0000000..1ac8265 --- /dev/null +++ b/acrypto/consumer.c @@ -0,0 +1,266 @@ +/* + * consumer.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef DEBUG +#include +#include +#include "crypto_user.h" + +#undef dprintk +//#define dprintk(f, a...) printk(f, ##a) +#define dprintk(f, a...) do {} while(0) + +#define KEY_LENGTH 16 +static char ckey[KEY_LENGTH]; +static char civ[KEY_LENGTH]; +static int key_length = sizeof(ckey); +static int iv_length = sizeof(civ); + +static void ctest_callback(struct crypto_session_initializer *ci, + struct crypto_data *data); + +static struct crypto_session_initializer ci = { + .operation = CRYPTO_OP_ENCRYPT, + .type = CRYPTO_TYPE_AES_128, + .mode = CRYPTO_MODE_CBC, + .priority = 4, + .callback = ctest_callback, +}; + +static struct crypto_data cdata; + +#define CSESSION_MAX 1000 +static struct crypto_session *s; +static atomic_t watermark; +static struct timer_list ctimer; +static char data_str[4096] = "test message qwerty"; + +static void ctest_callback(struct crypto_session_initializer *ci, + struct crypto_data *data) +{ + int i, off, size, ssize; + struct scatterlist *sg; + unsigned char *ptr; + + dprintk("%s: session %llu [%llu]\n", __func__, ci->id, ci->dev_id); + dprintk("src=%s, len=%d.\n", + ((char *)page_address(data->sg_src[0].page)) + data->sg_src[0].offset, + data->sg_src[0].length); +#if 1 + ssize = 32; + + sg = &data->sg_key[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("key[%d]=", size); + for (i = 0; i < size; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_src[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("src[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_dst[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("dst[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); +#endif + atomic_dec(&watermark); +} + +int ctimer_func(void *data, int size, int op) +{ + u8 *ptr; + + if (size > PAGE_SIZE) + size = PAGE_SIZE; + + ptr = kmap_atomic(cdata.sg_src[0].page, KM_USER1); + if (!ptr) + return -1; + + memcpy(ptr + cdata.sg_src[0].offset, data, size); + cdata.sg_src[0].length = size; + cdata.sg_dst[0].length = size; + + kunmap_atomic(ptr, KM_USER1); + + ci.operation = op; + s = crypto_session_alloc(&ci, &cdata); + if (s) + atomic_inc(&watermark); + + return (s) ? 0 : -EINVAL; +} + +static int alloc_sg(struct scatterlist *sg, void *data, int size) +{ + sg->offset = 0; + sg->page = alloc_pages(GFP_KERNEL, get_order(size)); + if (!sg->page) { + printk(KERN_ERR "Failed to allocate page.\n"); + return -ENOMEM; + } + + if (data) { + void *ptr = kmap(sg->page); + + if (!ptr) { + __free_pages(sg->page, get_order(size)); + return -ENOMEM; + } + + memcpy(ptr, data, size); + kunmap(ptr); + } + + sg->length = size; + + return 0; +} + +static void ctimerf(unsigned long data) +{ + int size, err, i; + + dprintk("%s started\n", __func__); + + size = 4096; + + for (i=0; i<1000; ++i) + err = ctimer_func(data_str, size, CRYPTO_OP_ENCRYPT); + //err = ctimer_func(data_str, size, CRYPTO_OP_DECRYPT); + + if (!err) + mod_timer(&ctimer, jiffies + 1); + else + mod_timer(&ctimer, jiffies + 1); + + dprintk("%s finished.\n", __func__); +} + +int consumer_init(void) +{ + int err; + + err = crypto_user_alloc_crypto_data(&cdata, sizeof(data_str), sizeof(data_str), key_length, iv_length); + if (err) + return err; + + err = alloc_sg(&cdata.sg_src[0], NULL, PAGE_SIZE); + if (err) + goto err_out_return; + + err = alloc_sg(&cdata.sg_dst[0], NULL, PAGE_SIZE); + if (err) + goto err_out_src; + + err = alloc_sg(&cdata.sg_key[0], ckey, key_length); + if (err) + goto err_out_dst; + + err = alloc_sg(&cdata.sg_iv[0], civ, iv_length); + if (err) + goto err_out_key; + + //cdata.priv_size = 256;//sizeof(struct aes_ctx); + + init_timer(&ctimer); + ctimer.function = &ctimerf; + ctimer.expires = jiffies + HZ; + ctimer.data = 0; + + add_timer(&ctimer); + + return 0; + + __free_pages(cdata.sg_iv[0].page, get_order(1)); +err_out_key: + __free_pages(cdata.sg_key[0].page, get_order(key_length)); +err_out_dst: + __free_pages(cdata.sg_dst[0].page, get_order(1)); +err_out_src: + __free_pages(cdata.sg_src[0].page, get_order(1)); +err_out_return: + crypto_user_free_crypto_data(&cdata); + + return -ENODEV; +} + +void consumer_fini(void) +{ + del_timer_sync(&ctimer); + + printk(KERN_INFO "%s: Timer has been removed.\n", __func__); + while (atomic_read(&watermark)) { + msleep(1000); + + printk(KERN_INFO "Waiting for sessions to be freed: watermark=%d.\n", + atomic_read(&watermark)); + } + + if (!cdata.sg_key[0].length) + printk("BUG: key length is 0 in %s.\n", __func__); + + __free_pages(cdata.sg_iv[0].page, get_order(1)); + __free_pages(cdata.sg_key[0].page, get_order(key_length)); + __free_pages(cdata.sg_dst[0].page, get_order(1)); + __free_pages(cdata.sg_src[0].page, get_order(1)); + + crypto_user_free_crypto_data(&cdata); + + printk(KERN_INFO "Test crypto module consumer is unloaded.\n"); +} + +module_init(consumer_init); +module_exit(consumer_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Test crypto module consumer."); diff --git a/acrypto/crypto_conn.c b/acrypto/crypto_conn.c new file mode 100644 index 0000000..cd4f1d1 --- /dev/null +++ b/acrypto/crypto_conn.c @@ -0,0 +1,135 @@ +/* + * crypto_conn.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" + +#include + +#include "crypto_conn.h" +#include "crypto_user_ioctl.h" +#include "crypto_user_direct.h" + +struct cb_id crypto_conn_id = { 0xdead, 0x0000 }; +static char crypto_conn_name[] = "crconn"; + +static void crypto_conn_callback(void *data) +{ + struct cn_msg *msg, *reply; + struct crypto_conn_data *d, *cmd; + struct crypto_device *dev; + + msg = (struct cn_msg *)data; + d = (struct crypto_conn_data *)msg->data; + + if (msg->len < sizeof(*d)) { + dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u < %zu.\n", + msg->len, sizeof(*d)); + return; + } + + if (msg->len != sizeof(*d) + d->len) { + dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u != %zu.\n", + msg->len, sizeof(*d) + d->len); + return; + } + + dev = crypto_device_get_name(d->name); + if (!dev) { + dprintk("Crypto device %s was not found.\n", d->name); + return; + } + + switch (d->cmd) { + case CRYPTO_GET_STAT: + reply = kmalloc(sizeof(*msg) + sizeof(*cmd) + sizeof(struct crypto_device_stat), GFP_ATOMIC); + if (reply) { + struct crypto_device_stat *ptr; + + memcpy(reply, msg, sizeof(*reply)); + reply->len = sizeof(*cmd) + sizeof(*ptr); + + /* + * See protocol description in connector.c + */ + reply->ack++; + + cmd = (struct crypto_conn_data *)(reply + 1); + memcpy(cmd, d, sizeof(*cmd)); + cmd->len = sizeof(*ptr); + + ptr = (struct crypto_device_stat *)(cmd + 1); + memcpy(ptr, &dev->stat, sizeof(*ptr)); + + cn_netlink_send(reply, 0, GFP_ATOMIC); + + kfree(reply); + } else + dprintk(KERN_ERR "Failed to allocate %zu bytes in reply to comamnd 0x%x.\n", + sizeof(*msg) + sizeof(*cmd), d->cmd); + break; + case CRYPTO_REQUEST: +#if 1 + { + struct crypto_user_direct *usr; + + usr = (struct crypto_user_direct *)(d->data); + + crypto_user_direct_add_request(msg->seq, msg->ack, usr); + } +#endif + break; + default: + dprintk(KERN_ERR "Wrong operation 0x%04x for crypto connector.\n", + d->cmd); + return; + } + + crypto_device_put(dev); +} + +int crypto_conn_init(void) +{ + int err; + + err = cn_add_callback(&crypto_conn_id, crypto_conn_name, crypto_conn_callback); + if (err) + return err; + + dprintk("Crypto connector callback is registered.\n"); + + return 0; +} + +void crypto_conn_fini(void) +{ + cn_del_callback(&crypto_conn_id); + dprintk("Crypto connector callback is unregistered.\n"); +} diff --git a/acrypto/crypto_conn.h b/acrypto/crypto_conn.h new file mode 100644 index 0000000..40bc3ac --- /dev/null +++ b/acrypto/crypto_conn.h @@ -0,0 +1,44 @@ +/* + * crypto_conn.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_CONN_H +#define __CRYPTO_CONN_H + +#include + +#define CRYPTO_REQUEST 1 +#define CRYPTO_GET_STAT 2 + +struct crypto_conn_data +{ + char name[SCACHE_NAMELEN]; + __u16 cmd; + __u16 len; + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +int crypto_conn_init(void); +void crypto_conn_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_CONN_H */ diff --git a/acrypto/crypto_dev.c b/acrypto/crypto_dev.c new file mode 100644 index 0000000..479f3ef --- /dev/null +++ b/acrypto/crypto_dev.c @@ -0,0 +1,507 @@ +/* + * crypto_dev.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +LIST_HEAD(cdev_list); +spinlock_t cdev_lock = SPIN_LOCK_UNLOCKED; +static u32 cdev_ids; +extern unsigned int max_ca_drivers; + +static int crypto_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int crypto_probe(struct device *dev) +{ + return -ENODEV; +} + +static int crypto_remove(struct device *dev) +{ + return 0; +} + +static void crypto_release(struct device *dev) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, device); + + complete(&d->dev_released); +} + +static void crypto_class_release(struct class *class) +{ +} + +static void crypto_class_release_device(struct class_device *class_dev) +{ +} + +struct class crypto_class = { + .name = "acrypto", + .class_release = crypto_class_release, + .release = crypto_class_release_device +}; + +struct bus_type crypto_bus_type = { + .name = "acrypto", + .match = crypto_match +}; + +struct device_driver crypto_driver = { + .name = "crypto_driver", + .bus = &crypto_bus_type, + .probe = crypto_probe, + .remove = crypto_remove, +}; + +struct device crypto_dev = { + .parent = NULL, + .bus = &crypto_bus_type, + .bus_id = "Asynchronous crypto", + .driver = &crypto_driver, + .release = &crypto_release +}; + +static ssize_t sessions_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%d\n", atomic_read(&d->refcnt)); +} +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%s\n", d->name); +} +static ssize_t devices_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(d, &cdev_list, cdev_entry) { + off += sprintf(buf + off, "%s ", d->name); + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!off) + off = sprintf(buf, "No devices registered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} + +static ssize_t queue_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + unsigned long flags; + struct crypto_queue *q; + int size = PAGE_SIZE, ret; + + spin_lock_irqsave(&d->queue.queue_lock, flags); + list_for_each_entry(q, &d->queue.queue_list, queue_entry) { + ret = snprintf(buf, size, "%4u - %u\n", q->priority, q->qlen); + size -= ret; + buf += ret; + } + spin_unlock_irqrestore(&d->queue.queue_lock, flags); + return PAGE_SIZE - size; +} +static ssize_t kmem_failed_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.kmem_failed); +} +static ssize_t sstarted_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sstarted); +} +static ssize_t sfinished_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sfinished); +} +static ssize_t scompleted_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.scompleted); +} + +static CLASS_DEVICE_ATTR(sessions, 0444, sessions_show, NULL); +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(devices, 0444, devices_show, NULL); +static CLASS_DEVICE_ATTR(scompleted, 0444, scompleted_show, NULL); +static CLASS_DEVICE_ATTR(sstarted, 0444, sstarted_show, NULL); +static CLASS_DEVICE_ATTR(sfinished, 0444, sfinished_show, NULL); +static CLASS_DEVICE_ATTR(kmem_failed, 0444, kmem_failed_show, NULL); +static CLASS_DEVICE_ATTR(queue, 0444, queue_show, NULL); + +static void create_device_attributes(struct crypto_device *dev) +{ + class_device_create_file(&dev->class_device, &class_device_attr_sessions); + class_device_create_file(&dev->class_device, &class_device_attr_name); + class_device_create_file(&dev->class_device, &class_device_attr_scompleted); + class_device_create_file(&dev->class_device, &class_device_attr_sstarted); + class_device_create_file(&dev->class_device, &class_device_attr_sfinished); + class_device_create_file(&dev->class_device, &class_device_attr_kmem_failed); + class_device_create_file(&dev->class_device, &class_device_attr_queue); +} + +static void remove_device_attributes(struct crypto_device *dev) +{ + class_device_remove_file(&dev->class_device, &class_device_attr_sessions); + class_device_remove_file(&dev->class_device, &class_device_attr_name); + class_device_remove_file(&dev->class_device, &class_device_attr_scompleted); + class_device_remove_file(&dev->class_device, &class_device_attr_sstarted); + class_device_remove_file(&dev->class_device, &class_device_attr_sfinished); + class_device_remove_file(&dev->class_device, &class_device_attr_kmem_failed); + class_device_remove_file(&dev->class_device, &class_device_attr_queue); +} + +int __match_initializer(struct crypto_capability *cap, struct crypto_session_initializer *ci, struct crypto_device *dev) +{ + unsigned int ref = atomic_read(&dev->refcnt) + 1; + int ret = 0; + + if (cap->operation == ci->operation && cap->type == ci->type && + cap->mode == (ci->mode & 0x1fff) && + cap->qlen >= ref) { + ret = 1; + dprintk("Match: %04x.%04x.%04x vs. %04x.%04x.%04x, max_len=%u, ref=%u.\n", + cap->operation, cap->type, cap->mode, + ci->operation, ci->type, ci->mode, + cap->qlen, ref); + } + + return ret; +} + +int match_initializer(struct crypto_device *dev, struct crypto_session_initializer *ci) +{ + int i; + + for (i = 0; i < dev->cap_number; ++i) { + struct crypto_capability *cap = &dev->cap[i]; + + if (__match_initializer(cap, ci, dev)) + return 1; + } + + return 0; +} + +void crypto_device_get(struct crypto_device *dev) +{ + atomic_inc(&dev->refcnt); +} + +struct crypto_device *crypto_device_get_name(char *name) +{ + struct crypto_device *dev; + int found = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (!strcmp(dev->name, name)) { + found = 1; + crypto_device_get(dev); + break; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!found) + return NULL; + + return dev; +} + +void crypto_device_put(struct crypto_device *dev) +{ + atomic_dec(&dev->refcnt); +} + +static int avg_cap_qlen(struct crypto_device *dev) +{ + int i, max = 0; + + if (!dev->cap_number) + return 0; + + for (i=0; icap_number; ++i) { + max += dev->cap[i].qlen; + } + + return (max / dev->cap_number); +} + +static int is_ca_index_taken(int index) +{ + struct crypto_device *dev; + unsigned long flags; + int result = 0; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if( dev->ca_index == index ) { + result = 1; + break; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + return result; +} + +int __crypto_device_add(struct crypto_device *dev) +{ + int err = -ENOMEM, avg; + + memset(&dev->stat, 0, sizeof(dev->stat)); + spin_lock_init(&dev->stat_lock); + spin_lock_init(&dev->lock); + atomic_set(&dev->refcnt, 0); + atomic_set(&dev->sid, 0); + init_completion(&dev->dev_released); + memcpy(&dev->device, &crypto_dev, sizeof(struct device)); + dev->driver = &crypto_driver; + INIT_LIST_HEAD(&dev->queue.queue_list); + spin_lock_init(&dev->queue.queue_lock); + + avg = avg_cap_qlen(dev); + if (!avg) + return -EINVAL; + + dev->ca_index = 0; /* ca_index is used only by context-aware drivers */ + if( dev->flags & CRYPTO_DEV_CTX_INIT) { + while ( is_ca_index_taken( dev->ca_index ) && + dev->ca_index < max_ca_drivers) + dev->ca_index++; + + if( dev->ca_index >= max_ca_drivers ) { + printk(KERN_ERR "Too many context-aware acrypto drivers (%d). " + "Failed to load driver %s.\n + Please reload acrypto with larger value of max_ca_drivers.\n", + max_ca_drivers, dev->name ); + return -EINVAL; + } + } + + dev->session_cache = kmem_cache_create(dev->name, sizeof(struct crypto_session), + 0, 0, NULL, NULL); + if (!dev->session_cache) { + dprintk(KERN_ERR "Failed to create session cache for device %s.\n", dev->name); + return err; + } + + dev->session_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->session_cache); + if (!dev->session_pool) { + dprintk(KERN_ERR "Failed to create session memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_cache_destroy; + } + + snprintf(dev->route_cache_name, sizeof(dev->route_cache_name), "%s-route", dev->name); + dev->route_cache = kmem_cache_create(dev->route_cache_name, sizeof(struct crypto_route), 0, 0, NULL, NULL); + if (!dev->route_cache) { + dprintk(KERN_ERR "Failed to create route cache for device %s.\n", dev->name); + goto err_out_mempool_destroy; + } + + avg = (avg)?avg*2:1024; + dev->route_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->route_cache); + if (!dev->route_pool) { + dprintk(KERN_ERR "Failed to create route memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_route_cache_destroy; + } + + snprintf(dev->queue_cache_name, sizeof(dev->queue_cache_name), "%s-queue", dev->name); + dev->queue_cache = kmem_cache_create(dev->queue_cache_name, sizeof(struct crypto_queue), 0, 0, NULL, NULL); + if (!dev->queue_cache) { + dprintk(KERN_ERR "Failed to create queue cache for device %s.\n", dev->name); + goto err_out_route_mempool_destroy; + } + + dev->queue_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->queue_cache); + if (!dev->queue_pool) { + dprintk(KERN_ERR "Failed to create queue memory pool with %d objects for device %s.\n", + avg, dev->name); + goto err_out_queue_cache_destroy; + } + + snprintf(dev->device.bus_id, sizeof(dev->device.bus_id), "%s", dev->name); + err = device_register(&dev->device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto device %s: err=%d.\n", + dev->name, err); + goto err_out_queue_mempool_destroy; + } + + snprintf(dev->class_device.class_id, sizeof(dev->class_device.class_id), "%s", dev->name); + dev->class_device.dev = &dev->device; + dev->class_device.class = &crypto_class; + + err = class_device_register(&dev->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto class device %s: err=%d.\n", + dev->name, err); + goto err_out_device_unregister; + } + + create_device_attributes(dev); + + return 0; + +err_out_device_unregister: + device_unregister(&dev->device); +err_out_queue_mempool_destroy: + mempool_destroy(dev->queue_pool); +err_out_queue_cache_destroy: + kmem_cache_destroy(dev->queue_cache); +err_out_route_mempool_destroy: + mempool_destroy(dev->route_pool); +err_out_route_cache_destroy: + kmem_cache_destroy(dev->route_cache); +err_out_mempool_destroy: + mempool_destroy(dev->session_pool); +err_out_cache_destroy: + kmem_cache_destroy(dev->session_cache); + + return err; +} + +void __crypto_device_remove(struct crypto_device *dev) +{ + remove_device_attributes(dev); + class_device_unregister(&dev->class_device); + device_unregister(&dev->device); + + crypto_destroy_queues(dev); + + mempool_destroy(dev->route_pool); + kmem_cache_destroy(dev->route_cache); + mempool_destroy(dev->session_pool); + kmem_cache_destroy(dev->session_cache); + mempool_destroy(dev->queue_pool); + kmem_cache_destroy(dev->queue_cache); +} + +int crypto_device_add(struct crypto_device *dev) +{ + int err; + unsigned long flags; + + err = __crypto_device_add(dev); + if (err) + return err; + + spin_lock_irqsave(&cdev_lock, flags); + list_add(&dev->cdev_entry, &cdev_list); + dev->id = ++cdev_ids; + spin_unlock_irqrestore(&cdev_lock, flags); + + printk("Crypto device %s was registered with ID=%x.\n", + dev->name, dev->id); + + return 0; +} + +void crypto_device_remove(struct crypto_device *dev) +{ + struct crypto_device *__dev, *n; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry_safe(__dev, n, &cdev_list, cdev_entry) { + if (compare_device(__dev, dev)) { + list_del_init(&__dev->cdev_entry); + spin_unlock_irqrestore(&cdev_lock, flags); + + + /* + * In test cases or when crypto device driver is not written correctly, + * it's ->data_ready() method will not be called anymore + * after device is removed from crypto device list. + * + * For such cases we either should provide ->flush() call + * or properly write ->data_ready() method. + */ + + while (atomic_read(&__dev->refcnt)) { + + dprintk("Waiting for %s to become free: refcnt=%d.\n", + __dev->name, atomic_read(&dev->refcnt)); + + /* + * Hack zone: you need to write good ->data_ready() + * and crypto device driver itself. + * + * Driver shoud not buzz if it has pending sessions + * in it's queue and it was removed from global device list. + * + * Although I can workaround it here, for example by + * flushing the whole queue and drop all pending sessions. + */ + + __dev->data_ready(__dev); + ssleep(1); + } + + __crypto_device_remove(dev); + + dprintk(KERN_ERR "Crypto device %s was unregistered.\n", + dev->name); + return; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + dprintk(KERN_ERR "Crypto device %s was not registered.\n", dev->name); +} + +EXPORT_SYMBOL_GPL(crypto_device_add); +EXPORT_SYMBOL_GPL(crypto_device_remove); +EXPORT_SYMBOL_GPL(crypto_device_get); +EXPORT_SYMBOL_GPL(crypto_device_get_name); +EXPORT_SYMBOL_GPL(crypto_device_put); +EXPORT_SYMBOL_GPL(match_initializer); diff --git a/acrypto/crypto_lb.c b/acrypto/crypto_lb.c new file mode 100644 index 0000000..498e2ad --- /dev/null +++ b/acrypto/crypto_lb.c @@ -0,0 +1,504 @@ +/* + * crypto_lb.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" +#include "crypto_stat.h" +#include "crypto_route.h" + +static LIST_HEAD(crypto_lb_list); +static spinlock_t crypto_lb_lock = SPIN_LOCK_UNLOCKED; +static int lb_num = 0; +static struct crypto_lb *current_lb, *default_lb; +static struct workqueue_struct *crypto_lb_queue; + +extern struct list_head cdev_list; +extern spinlock_t cdev_lock; + +extern struct crypto_lb simple_lb; + +static int lb_is_current(struct crypto_lb *l) +{ + return (l == current_lb); +} + +static int lb_is_default(struct crypto_lb *l) +{ + return (l == default_lb); +} + +#if 0 +static void lb_set_current(struct crypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + current_lb = l; + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} + +static void lb_set_default(struct crypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + default_lb = l; + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} +#endif +static int crypto_lb_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int crypto_lb_probe(struct device *dev) +{ + return -ENODEV; +} + +static int crypto_lb_remove(struct device *dev) +{ + return 0; +} + +static void crypto_lb_release(struct device *dev) +{ + struct crypto_lb *d = container_of(dev, struct crypto_lb, device); + + complete(&d->dev_released); +} + +static void crypto_lb_class_release(struct class *class) +{ +} + +static void crypto_lb_class_release_device(struct class_device *class_dev) +{ +} + +struct class crypto_lb_class = { + .name = "crypto_lb", + .class_release = crypto_lb_class_release, + .release = crypto_lb_class_release_device +}; + +struct bus_type crypto_lb_bus_type = { + .name = "crypto_lb", + .match = crypto_lb_match +}; + +struct device_driver crypto_lb_driver = { + .name = "crypto_lb_driver", + .bus = &crypto_lb_bus_type, + .probe = crypto_lb_probe, + .remove = crypto_lb_remove, +}; + +struct device crypto_lb_dev = { + .parent = NULL, + .bus = &crypto_lb_bus_type, + .bus_id = "crypto load balancer", + .driver = &crypto_lb_driver, + .release = &crypto_lb_release +}; + +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct crypto_lb *lb = container_of(dev, struct crypto_lb, class_device); + + return sprintf(buf, "%s\n", lb->name); +} + +static ssize_t current_show(struct class_device *dev, char *buf) +{ + struct crypto_lb *lb; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(lb, &crypto_lb_list, lb_entry) { + if (lb_is_current(lb)) + off += sprintf(buf + off, "["); + if (lb_is_default(lb)) + off += sprintf(buf + off, "("); + off += sprintf(buf + off, "%s", lb->name); + if (lb_is_default(lb)) + off += sprintf(buf + off, ")"); + if (lb_is_current(lb)) + off += sprintf(buf + off, "]"); + } + + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + if (!off) + off = sprintf(buf, "No load balancers regitered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} +static ssize_t current_store(struct class_device *dev, const char *buf, size_t count) +{ + struct crypto_lb *lb; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(lb, &crypto_lb_list, lb_entry) { + if (count == strlen(lb->name) && !strcmp(buf, lb->name)) { + current_lb = lb; + default_lb = lb; + + dprintk("Load balancer %s is set as current and default.\n", + lb->name); + + break; + } + } + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + return count; +} + +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(lbs, 0644, current_show, current_store); + +static void create_device_attributes(struct crypto_lb *lb) +{ + class_device_create_file(&lb->class_device, &class_device_attr_name); +} + +static void remove_device_attributes(struct crypto_lb *lb) +{ + class_device_remove_file(&lb->class_device, &class_device_attr_name); +} + +static int compare_lb(struct crypto_lb *l1, struct crypto_lb *l2) +{ + if (!strncmp(l1->name, l2->name, sizeof(l1->name))) + return 1; + + return 0; +} + +struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_device *dev; + unsigned long flags; + + if (!current_lb) + return NULL; + + if (sci_bound(ci)) { + int found = 0; + + spin_lock_irqsave(&cdev_lock, flags); + + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->id == ci->bdev) { + found = 1; + break; + } + } + + spin_unlock_irqrestore(&cdev_lock, flags); + + return (found) ? dev : NULL; + } + + spin_lock_irqsave(¤t_lb->lock, flags); + dev = current_lb->find_device(current_lb, ci, data); + spin_unlock_irqrestore(¤t_lb->lock, flags); + + return dev; +} + +static int __crypto_lb_register(struct crypto_lb *lb) +{ + int err; + + spin_lock_init(&lb->lock); + + init_completion(&lb->dev_released); + memcpy(&lb->device, &crypto_lb_dev, sizeof(struct device)); + lb->driver = &crypto_lb_driver; + + snprintf(lb->device.bus_id, sizeof(lb->device.bus_id), "%s", lb->name); + err = device_register(&lb->device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer device %s: err=%d.\n", + lb->name, err); + return err; + } + + snprintf(lb->class_device.class_id, sizeof(lb->class_device.class_id), "%s", lb->name); + lb->class_device.dev = &lb->device; + lb->class_device.class = &crypto_lb_class; + + err = class_device_register(&lb->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer class device %s: err=%d.\n", + lb->name, err); + device_unregister(&lb->device); + return err; + } + + create_device_attributes(lb); + + return 0; + +} + +void __crypto_lb_unregister(struct crypto_lb *lb) +{ + remove_device_attributes(lb); + class_device_unregister(&lb->class_device); + device_unregister(&lb->device); +} + +int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default) +{ + struct crypto_lb *__lb; + int err; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(__lb, &crypto_lb_list, lb_entry) { + if (unlikely(compare_lb(__lb, lb))) { + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + dprintk(KERN_ERR "Crypto load balancer %s is already registered.\n", + lb->name); + return -EINVAL; + } + } + + list_add(&lb->lb_entry, &crypto_lb_list); + + if (!default_lb || set_default) + default_lb = lb; + + if (!current_lb || set_current) + current_lb = lb; + + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + err = __crypto_lb_register(lb); + if (err) { + spin_lock_irqsave(&crypto_lb_lock, flags); + list_del_init(&lb->lb_entry); + current_lb = &simple_lb; + default_lb = &simple_lb; + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + return err; + } + + dprintk("Crypto load balancer %s was registered and set to be [%s.%s].\n", + lb->name, (lb_is_current(lb)) ? "current" : "not current", + (lb_is_default(lb)) ? "default" : "not default"); + + lb_num++; + + return 0; +} + +void crypto_lb_unregister(struct crypto_lb *lb) +{ + struct crypto_lb *__lb, *n; + unsigned long flags; + + __crypto_lb_unregister(lb); + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry_safe(__lb, n, &crypto_lb_list, lb_entry) { + if (compare_lb(__lb, lb)) { + lb_num--; + list_del_init(&__lb->lb_entry); + + dprintk("Crypto load balancer %s was unregistered.\n", lb->name); + break; + } + } + + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} + +static void crypto_lb_queue_wrapper(void *data) +{ + struct crypto_session *s = (struct crypto_session *)data; + + dprintk("%s: Calling callback for session %08x [%08x] flags=%lx, " + "op=%04u, type=%04x, mode=%04x, priority=%04x\n", __func__, + s->ci.id, s->ci.dev_id, s->ci.flags, s->ci.operation, + s->ci.type, s->ci.mode, s->ci.priority); + + s->ci.callback(&s->ci, &s->data); + + if (likely(session_completed(s))) { + crypto_session_destroy(s); + return; + } else { + /* + * Not implemented yet. + */ + /* + * Special case: crypto consumer marks session as "not finished" + * in it's callback - it means that crypto consumer wants + * this session to be processed further, + * for example crypto consumer can add new route and then + * mark session as "not finished". + */ + BUG(); + } +} + +static void crypto_lb_process_next_route(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_copy_current_ci(s); + if (dev) + __crypto_session_add(dev, s); +} + +int complete_session(struct crypto_session *s) +{ + struct crypto_device *cdev; + + cdev = crypto_route_get_current_device(s); + + crypto_stat_complete_inc(s); + crypto_stat_ptime_inc(s); + crypto_session_dequeue_route(s); + set_bit(SESSION_COMPLETED, &s->ci.flags); + + if (session_broken(s)) { + struct crypto_device *ldev; + int err; + + unbreak_session(s); + ldev = crypto_lb_find_device(&s->ci, &s->data); + if (ldev && ldev != cdev) { + err = crypto_route_add_direct(ldev, s, &s->ci); + if (!err) { + err = crypto_session_add(s); + if (!err) + return 0; + } + } else if (ldev) + crypto_device_put(ldev); + break_session(s); + } + + if (unlikely(crypto_route_queue_len(s))) { + crypto_lb_process_next_route(s); + } else { + if (test_bit(SESSION_DIRECT, &s->ci.flags)) + crypto_lb_queue_wrapper(s); + else { + INIT_WORK(&s->work, &crypto_lb_queue_wrapper, s); + queue_work(crypto_lb_queue, &s->work); + } + } + + return 0; +} + +int crypto_lb_init(void) +{ + int err; + + err = bus_register(&crypto_lb_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer bus: err=%d.\n", err); + goto err_out_exit; + } + + err = driver_register(&crypto_lb_driver); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer driver: err=%d.\n", err); + goto err_out_bus_unregister; + } + + crypto_lb_class.class_dev_attrs = &class_device_attr_lbs; + + err = class_register(&crypto_lb_class); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer class: err=%d.\n", err); + goto err_out_driver_unregister; + } + + crypto_lb_queue = create_singlethread_workqueue("clbq"); + if (!crypto_lb_queue) { + dprintk(KERN_ERR "Failed to create crypto load balaner work queue.\n"); + goto err_out_class_unregister; + } + + err = crypto_lb_register(&simple_lb, 1, 1); + if (err) + goto err_out_destroy_workqueue; + + return 0; + +err_out_destroy_workqueue: + destroy_workqueue(crypto_lb_queue); +err_out_class_unregister: + class_unregister(&crypto_lb_class); +err_out_driver_unregister: + driver_unregister(&crypto_lb_driver); +err_out_bus_unregister: + bus_unregister(&crypto_lb_bus_type); +err_out_exit: + return err; +} + +void crypto_lb_fini(void) +{ + __crypto_lb_unregister(&simple_lb); + + flush_workqueue(crypto_lb_queue); + destroy_workqueue(crypto_lb_queue); + class_unregister(&crypto_lb_class); + driver_unregister(&crypto_lb_driver); + bus_unregister(&crypto_lb_bus_type); +} + +EXPORT_SYMBOL_GPL(crypto_lb_register); +EXPORT_SYMBOL_GPL(crypto_lb_unregister); +EXPORT_SYMBOL_GPL(crypto_lb_find_device); +EXPORT_SYMBOL_GPL(complete_session); diff --git a/acrypto/crypto_lb.h b/acrypto/crypto_lb.h new file mode 100644 index 0000000..1539d45 --- /dev/null +++ b/acrypto/crypto_lb.h @@ -0,0 +1,56 @@ +/* + * crypto_lb.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_LB_H +#define __CRYPTO_LB_H + +#include + +#define CRYPTO_LB_NAMELEN 32 + +struct crypto_lb +{ + struct list_head lb_entry; + + char name[CRYPTO_LB_NAMELEN]; + + struct crypto_device * (*find_device) (struct crypto_lb *, + struct crypto_session_initializer *, + struct crypto_data *); + + spinlock_t lock; + + struct device_driver *driver; + struct device device; + struct class_device class_device; + struct completion dev_released; + +}; + +int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default); +void crypto_lb_unregister(struct crypto_lb *); + +struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *, struct crypto_data *); + +int crypto_lb_init(void); +void crypto_lb_fini(void); + +#endif /* __CRYPTO_LB_H */ diff --git a/acrypto/crypto_main.c b/acrypto/crypto_main.c new file mode 100644 index 0000000..ced7008 --- /dev/null +++ b/acrypto/crypto_main.c @@ -0,0 +1,476 @@ +/* + * crypto_main.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" +#include "crypto_conn.h" +#include "crypto_route.h" +#include "crypto_user_ioctl.h" +#include "crypto_stat.h" + +unsigned int max_ca_drivers = 16; /* max # of context-aware drivers */ +module_param(max_ca_drivers, uint, 0); + +extern struct bus_type crypto_bus_type; +extern struct device_driver crypto_driver; +extern struct class crypto_class; +extern struct device crypto_dev; +extern spinlock_t cdev_lock; +extern struct list_head cdev_list; + +static atomic_t global_session_id = ATOMIC_INIT(0); +static int acrypto_use_connector; +static DEFINE_MUTEX(context_mutex); +static LIST_HEAD(context_list); + +void crypto_session_destroy(struct crypto_session *s) +{ + if (s->data.priv_size && s->data.priv) + kfree(s->data.priv); + + if (session_from_cache(s)) + kmem_cache_free(s->pool_dev->session_cache, s); + else + mempool_free(s, s->pool_dev->session_pool); +} + +struct crypto_session *crypto_session_create(struct crypto_session_initializer *ci, struct crypto_data *d) +{ + struct crypto_device *ldev; + struct crypto_session *s; + int err; + + if (d->priv_size > CRYPTO_MAX_PRIV_SIZE) { + dprintk("priv_size %u is too big, maximum allowed %u.\n", + d->priv_size, CRYPTO_MAX_PRIV_SIZE); + return NULL; + } + + /* + * We grab a reference here if we can find device. + * This reference will be dropped when route will be removed. + */ + ldev = crypto_lb_find_device(ci, d); + if (!ldev) { + dprintk("Cannot find suitable device for [%02x.%02x.%02x.%02x].\n", + ci->operation, ci->mode, ci->type, ci->priority); + return NULL; + } + + s = mempool_alloc(ldev->session_pool, GFP_ATOMIC); + if (!s) { + ldev->stat.pool_failed++; + + s = kmem_cache_alloc(ldev->session_cache, GFP_ATOMIC); + if (!s) { + ldev->stat.kmem_failed++; + goto err_out_device_put; + } + + mark_session_from_cache(s); + } + + s->pool_dev = ldev; + + crypto_route_head_init(&s->route_list); + INIT_LIST_HEAD(&s->dev_queue_entry); + + spin_lock_init(&s->lock); + + memcpy(&s->ci, ci, sizeof(s->ci)); + memcpy(&s->data, d, sizeof(s->data)); + + s->data.priv = NULL; + if (d->priv_size) { + s->data.priv = kmalloc(d->priv_size, GFP_ATOMIC); + if (!s->data.priv) { + ldev->stat.kmem_failed++; + goto err_out_session_free; + } + + if (d->priv) + memcpy(s->data.priv, d->priv, d->priv_size); + } + else + s->data.priv = d->priv; + + s->ci.id = atomic_inc_return(&global_session_id); + s->ci.dev_id = atomic_inc_return(&ldev->sid); + s->ci.flags = 0; + + err = crypto_route_add_direct(ldev, s, ci); + if (err) { + ldev->stat.route_failed++; + dprintk("Can not add route to device %s.\n", ldev->name); + goto err_out_session_free; + } + + return s; + +err_out_session_free: + crypto_session_destroy(s); +err_out_device_put: + crypto_device_put(ldev); + + return NULL; +} + +int __crypto_session_add(struct crypto_device *ldev, struct crypto_session *s) +{ + int err; + + err = crypto_session_enqueue(ldev, s); + + if (!err && ldev->data_ready) + ldev->data_ready(ldev); + + return err; +} + +int crypto_session_add(struct crypto_session *s) +{ + struct crypto_device *ldev; + int err; + + ldev = crypto_route_get_current_device(s); + if (!ldev) + return -ENODEV; + + err = __crypto_session_add(ldev, s); + if (err) + /* + * Decrement usage counter, which was gotten in lookup() method. + */ + crypto_device_put(ldev); + + return err; +} + +struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *ci, struct crypto_data *d) +{ + struct crypto_session *s; + int err; + + s = crypto_session_create(ci, d); + if (!s) + return NULL; + + err = crypto_session_add(s); + if (err) { + crypto_session_destroy(s); + return NULL; + } + + return s; +} + +int crypto_session_dequeue_route(struct crypto_session *s) +{ + struct crypto_route *rt; + + rt = crypto_route_dequeue(s); + if (rt) { + crypto_route_free(rt); + return 0; + } + + return -ENODEV; +} + +void crypto_process(struct crypto_session *s, + int (*proc)(struct crypto_session *s, void *src, int slen, void *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + int (*proc_scatter)(struct crypto_session *s, struct scatterlist *src, + struct scatterlist *dst, int nbytes, void *key, int klen, void *iv, int ilen, void *data), + void (*finish)(struct crypto_session *s, void *key, int klen, void *iv, int ilen, void *data), + void *data) +{ + int i, keylen, ivlen, err = -EINVAL; + void *key, *iv; + + if (session_completed(s)) + return; + + key = iv = NULL; + ivlen = keylen = 0; + + err = -ENOMEM; + /* + * Simple case - key is small(it's size is less than PAGE_SIZE). + * Assymetric crypto will require proper key sg handling. + */ + if (s->data.sg_key[0].length) { + key = kmap(s->data.sg_key[0].page); + if (!key) + goto err_out_exit; + key += s->data.sg_key[0].offset; + keylen = s->data.sg_key[0].length; + } + + if (s->data.sg_iv[0].length) { + iv = kmap(s->data.sg_iv[0].page); + if (!iv) + goto err_out_unmap_key; + iv += s->data.sg_iv[0].offset; + ivlen = s->data.sg_iv[0].length; + } + + if (proc) { + for (i=0; idata.sg_src_num; ++i) { + int dlen, slen; + u8 *dst, *src; + + slen = s->data.sg_src[i].length; + dlen = s->data.sg_dst[i].length; + + + dst = kmap_atomic(s->data.sg_dst[i].page, KM_USER0) + s->data.sg_dst[i].offset; + src = kmap_atomic(s->data.sg_src[i].page, KM_USER1) + s->data.sg_src[i].offset; + + err = proc(s, src, slen, dst, dlen, key, keylen, iv, ivlen, data); + + kunmap_atomic(src, KM_USER1); + kunmap_atomic(dst, KM_USER0); + + s->data.sg_dst[i].length = s->data.sg_src[i].length; + s->data.sg_dst[i].offset = s->data.sg_src[i].offset; + } + } else if (proc_scatter) { + int nbytes = 0; + + for (i=0; idata.sg_src_num; ++i) + nbytes += s->data.sg_src[i].length; + err = proc_scatter(s, s->data.sg_src, s->data.sg_dst, nbytes, key, keylen, iv, ivlen, data); + } else + err = -EINVAL; + + if (err < 0) { + break_session(s); + if (printk_ratelimit()) + printk("acrypto: operation=%02x, proc=%p, proc_scatter=%p, err=%d.\n", + s->ci.operation, proc, proc_scatter, err); + } + + if (finish) + finish(s, key, keylen, iv, ivlen, data); + + if (iv) + kunmap(s->data.sg_iv[0].page); +err_out_unmap_key: + if (key) + kunmap(s->data.sg_key[0].page); +err_out_exit: + + complete_session(s); +} + +int crypto_context_register(struct crypto_context *ctx) +{ + unsigned long flags; + struct crypto_device *dev; + + mutex_lock(&context_mutex); + list_add_tail(&ctx->entry, &context_list); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->flags & CRYPTO_DEV_CTX_INIT) + dev->context_callback(dev, ctx, CRYPTO_DEV_CTX_INIT); + } + spin_unlock_irqrestore(&cdev_lock, flags); + mutex_unlock(&context_mutex); + + return 0; +} + +void crypto_context_unregister(struct crypto_context *ctx) +{ + unsigned long flags; + struct crypto_device *dev; + + mutex_lock(&context_mutex); + list_add_tail(&ctx->entry, &context_list); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->flags & CRYPTO_DEV_CTX_EXIT) + dev->context_callback(dev, ctx, CRYPTO_DEV_CTX_EXIT); + } + spin_unlock_irqrestore(&cdev_lock, flags); + mutex_unlock(&context_mutex); +} + +void crypto_context_notify(struct crypto_context *ctx, u32 ctx_flags) +{ + unsigned long flags; + struct crypto_device *dev; + + mutex_lock(&context_mutex); + list_add_tail(&ctx->entry, &context_list); + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if ((dev->flags & ctx_flags) && dev->context_callback) + dev->context_callback(dev, ctx, ctx_flags); + } + spin_unlock_irqrestore(&cdev_lock, flags); + mutex_unlock(&context_mutex); +} + +struct crypto_context *crypto_context_alloc(u16 type, u16 mode, u8 *key, unsigned int key_size) +{ + struct crypto_context *ctx; + int err; + int driver_priv_size; + + driver_priv_size = sizeof(void*) * max_ca_drivers; + ctx = kzalloc(sizeof(struct crypto_context) + driver_priv_size, GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->type = type; + ctx->mode = mode; + ctx->key_size = key_size; + + ctx->key = kmalloc(key_size, GFP_KERNEL); + if (!ctx->key) { + kfree(ctx); + return NULL; + } + memcpy(ctx->key, key, key_size); + atomic_set(&ctx->refcnt, 1); + + err = crypto_context_register(ctx); + if (err) { + kfree(ctx->key); + kfree(ctx); + return NULL; + } + + return ctx; +} + +void crypto_context_free(struct crypto_context *ctx) +{ + crypto_context_unregister(ctx); + kfree(ctx->key); + kfree(ctx); +} + +int cmain_init(void) +{ + int err; + + err = bus_register(&crypto_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register crypto bus: err=%d.\n", + err); + return err; + } + + err = driver_register(&crypto_driver); + if (err) { + dprintk(KERN_ERR "Failed to register crypto driver: err=%d.\n", + err); + goto err_out_bus_unregister; + } + + err = class_register(&crypto_class); + if (err) { + dprintk(KERN_ERR "Failed to register crypto class: err=%d.\n", + err); + goto err_out_driver_unregister; + } + + err = crypto_lb_init(); + if (err) + goto err_out_class_unregister; + + acrypto_use_connector = 0; +#if 0 + if (cn_already_initialized) { + err = crypto_conn_init(); + if (!err) + acrypto_use_connector = 1; + } +#endif + if (!acrypto_use_connector) + printk(KERN_INFO "%s: Connector has not initialized yet, support disabled.\n", __func__); + + err = crypto_user_ioctl_init(); + if (err) + goto err_out_crypto_conn_fini; + + return 0; + +err_out_crypto_conn_fini: + if (acrypto_use_connector) + crypto_conn_fini(); + crypto_lb_fini(); +err_out_class_unregister: + class_unregister(&crypto_class); +err_out_driver_unregister: + driver_unregister(&crypto_driver); +err_out_bus_unregister: + bus_unregister(&crypto_bus_type); + + return err; +} + +void cmain_fini(void) +{ + crypto_user_ioctl_fini(); + + if (acrypto_use_connector) + crypto_conn_fini(); + crypto_lb_fini(); + + class_unregister(&crypto_class); + driver_unregister(&crypto_driver); + bus_unregister(&crypto_bus_type); +} + +module_init(cmain_init); +module_exit(cmain_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Asynchronous crypto layer."); + +EXPORT_SYMBOL(crypto_session_alloc); +EXPORT_SYMBOL_GPL(crypto_session_create); +EXPORT_SYMBOL_GPL(crypto_session_add); +EXPORT_SYMBOL_GPL(__crypto_session_add); +EXPORT_SYMBOL_GPL(crypto_process); +EXPORT_SYMBOL_GPL(crypto_session_dequeue_route); +EXPORT_SYMBOL_GPL(crypto_context_alloc); +EXPORT_SYMBOL_GPL(crypto_context_free); +EXPORT_SYMBOL_GPL(crypto_context_register); +EXPORT_SYMBOL_GPL(crypto_context_unregister); +EXPORT_SYMBOL_GPL(crypto_context_notify); diff --git a/acrypto/crypto_queue.c b/acrypto/crypto_queue.c new file mode 100644 index 0000000..77d4702 --- /dev/null +++ b/acrypto/crypto_queue.c @@ -0,0 +1,168 @@ +/* + * crypto_queue.c + * + * Copyright (c) 2005 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static struct crypto_session *crypto_dequeue_one(struct crypto_queue *q) +{ + struct crypto_session *s = NULL; + + if (q->qlen) { + s = list_entry(q->session_list.next, struct crypto_session, dev_queue_entry); + list_del(&s->dev_queue_entry); + q->qlen--; + } + + return s; +} + +static int crypto_enqueue_one(struct crypto_queue *q, struct crypto_session *s) +{ + list_add_tail(&s->dev_queue_entry, &q->session_list); + q->qlen++; + + return 0; +} + +static struct crypto_queue *crypto_create_queue(struct crypto_device *dev, u16 priority) +{ + struct crypto_queue *q, *n; + struct list_head *prev; + int found = 0; + + q = mempool_alloc(dev->queue_pool, GFP_ATOMIC); + if (!q) + return NULL; + + INIT_LIST_HEAD(&q->session_list); + q->qlen = 0; + q->priority = priority; + + prev = &dev->queue.queue_list; + list_for_each_entry(n, &dev->queue.queue_list, queue_entry) { + if (n->priority > priority) { + list_add(&q->queue_entry, prev); + dev->queue.num++; + found = 1; + break; + } + prev = &n->queue_entry; + } + + if (!found) { + list_add(&q->queue_entry, &dev->queue.queue_list); + dev->queue.num++; + return q; + } + + + return q; +} + +static void crypto_queue_flush(struct crypto_queue *q) +{ + struct crypto_session *s; + + while ((s = crypto_dequeue_one(q)) != NULL) + crypto_session_destroy(s); +} + +void crypto_destroy_queues(struct crypto_device *dev) +{ + unsigned long flags; + struct crypto_queue *q, *n; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + list_for_each_entry_safe(q, n, &dev->queue.queue_list, queue_entry) { + list_del(&q->queue_entry); + dev->queue.num--; + + crypto_queue_flush(q); + + mempool_free(q, dev->queue_pool); + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); +} + +int crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s) +{ + int err = -EINVAL, found = 0; + unsigned long flags; + struct crypto_queue *q; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + list_for_each_entry(q, &dev->queue.queue_list, queue_entry) { + if (q->priority == s->ci.priority) { + err = crypto_enqueue_one(q, s); + found = 1; + break; + } + } + + if (!found) { + q = crypto_create_queue(dev, s->ci.priority); + if (q) + err = crypto_enqueue_one(q, s); + else + err = -ENOMEM; + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); + + return err; +} + +struct crypto_session *crypto_session_dequeue(struct crypto_device *dev) +{ + unsigned long flags; + struct crypto_session *s = NULL; + struct crypto_queue *q; + + spin_lock_irqsave(&dev->queue.queue_lock, flags); + if (dev->queue.num) { + list_for_each_entry(q, &dev->queue.queue_list, queue_entry) { + if (q->qlen) { + s = crypto_dequeue_one(q); + if (s) + break; + } + } + } + spin_unlock_irqrestore(&dev->queue.queue_lock, flags); + + return s; +} + +EXPORT_SYMBOL_GPL(crypto_session_dequeue); +EXPORT_SYMBOL_GPL(crypto_destroy_queues); diff --git a/acrypto/crypto_route.h b/acrypto/crypto_route.h new file mode 100644 index 0000000..aee7e23 --- /dev/null +++ b/acrypto/crypto_route.h @@ -0,0 +1,273 @@ +/* + * crypto_route.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_ROUTE_H +#define __CRYPTO_ROUTE_H + +#include +#include +#include + +#include + +static inline struct crypto_route *crypto_route_alloc_direct(struct crypto_device *dev, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = mempool_alloc(dev->route_pool, GFP_ATOMIC); + if (!rt) + return NULL; + + memset(rt, 0, sizeof(*rt)); + memcpy(&rt->ci, ci, sizeof(*ci)); + + rt->dev = dev; + + return rt; +} + +static inline struct crypto_route *crypto_route_alloc(struct crypto_device *dev, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + if (!match_initializer(dev, ci)) + return NULL; + + rt = crypto_route_alloc_direct(dev, ci); + + return rt; +} + +/* + * Device's reference counter must be incremented before routing was + * allocated, so routing subsystem itself does not care about it. + */ +static inline void crypto_route_free(struct crypto_route *rt) +{ + struct crypto_device *dev = rt->dev; + + mempool_free(rt, dev->route_pool); + crypto_device_put(dev); +} + +static inline void __crypto_route_del(struct crypto_route *rt, struct crypto_route_head *list) +{ + struct crypto_route *next, *prev; + + list->qlen--; + next = rt->next; + prev = rt->prev; + rt->next = rt->prev = NULL; + rt->list = NULL; + next->prev = prev; + prev->next = next; +} + +static inline void crypto_route_del(struct crypto_route *rt) +{ + struct crypto_route_head *list = rt->list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + if (list == rt->list) + __crypto_route_del(rt, rt->list); + spin_unlock_irqrestore(&list->lock, flags); + + crypto_route_free(rt); + } +} + +static inline struct crypto_route *__crypto_route_dequeue(struct crypto_route_head *list) +{ + struct crypto_route *next, *prev, *result; + + prev = (struct crypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) { + result = next; + next = next->next; + list->qlen--; + next->prev = prev; + prev->next = next; + result->next = result->prev = NULL; + result->list = NULL; + } + return result; +} + +static inline struct crypto_route *crypto_route_dequeue(struct crypto_session *s) +{ + struct crypto_route *rt; + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + rt = __crypto_route_dequeue(&s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); + + return rt; +} + +static inline void __crypto_route_queue(struct crypto_route *rt, struct crypto_route_head *list) +{ + struct crypto_route *prev, *next; + + rt->list = list; + list->qlen++; + next = (struct crypto_route *)list; + prev = next->prev; + rt->next = next; + rt->prev = prev; + next->prev = prev->next = rt; +} + +static inline void crypto_route_queue(struct crypto_route *rt, struct crypto_session *s) +{ + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + __crypto_route_queue(rt, &s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); +} + +/* + * Caller must hold a reference for crypto_device being added, + * and it is not permitted to drop the reference until routing is removed. + * See crypto_session_create() for example of right device's reference counter usage. + */ + +static inline int crypto_route_add(struct crypto_device *dev, struct crypto_session *s, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = crypto_route_alloc(dev, ci); + if (!rt) + return -ENOMEM; + + crypto_route_queue(rt, s); + + return 0; +} + +static inline int crypto_route_add_direct(struct crypto_device *dev, struct crypto_session *s, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = crypto_route_alloc_direct(dev, ci); + if (!rt) + return -ENOMEM; + + crypto_route_queue(rt, s); + + return 0; +} + +static inline int crypto_route_queue_len(struct crypto_session *s) +{ + return s->route_list.qlen; +} + +static inline void crypto_route_head_init(struct crypto_route_head *list) +{ + spin_lock_init(&list->lock); + list->prev = list->next = (struct crypto_route *)list; + list->qlen = 0; +} + +static inline struct crypto_route *__crypto_route_current(struct crypto_route_head *list) +{ + struct crypto_route *next, *prev, *result; + + prev = (struct crypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) + result = next; + + return result; +} + +static inline struct crypto_route *crypto_route_current(struct crypto_session *s) +{ + struct crypto_route_head *list; + struct crypto_route *rt = NULL; + + list = &s->route_list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __crypto_route_current(list); + spin_unlock_irqrestore(&list->lock, flags); + } + + return rt; +} + +static inline struct crypto_device *crypto_route_copy_current_ci(struct crypto_session *s) +{ + struct crypto_route *rt; + struct crypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + + rt = __crypto_route_current(list); + if (rt) { + memcpy(&s->ci, &rt->ci, sizeof(s->ci)); + return rt->dev; + } + + spin_unlock_irqrestore(&list->lock, flags); + + return NULL; +} + +/* + * It is not allowed to use returned device pointer + * after current route is removed. + * If you want to use current device and remove current route, + * you MUST grab the reference before removing the route. + */ +static inline struct crypto_device *crypto_route_get_current_device(struct crypto_session *s) +{ + struct crypto_route *rt; + struct crypto_device *dev = NULL; + struct crypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __crypto_route_current(list); + if (rt) + dev = rt->dev; + spin_unlock_irqrestore(&list->lock, flags); + + return dev; +} + +#endif /* __CRYPTO_ROUTE_H */ diff --git a/acrypto/crypto_stat.c b/acrypto/crypto_stat.c new file mode 100644 index 0000000..a6f5fe0 --- /dev/null +++ b/acrypto/crypto_stat.c @@ -0,0 +1,69 @@ +/* + * crypto_stat.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_route.h" + +void crypto_stat_complete_inc(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_get_current_device(s); + if (dev) { + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + dev->stat.scompleted++; + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +void crypto_stat_ptime_inc(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_get_current_device(s); + if (dev) { + int i; + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + for (i = 0; i < dev->cap_number; ++i) { + if (__match_initializer(&dev->cap[i], &s->ci, dev)) { + dev->cap[i].ptime += s->ci.ptime; + dev->cap[i].scomp++; + break; + } + } + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +EXPORT_SYMBOL(crypto_stat_complete_inc); diff --git a/acrypto/crypto_stat.h b/acrypto/crypto_stat.h new file mode 100644 index 0000000..4894b44 --- /dev/null +++ b/acrypto/crypto_stat.h @@ -0,0 +1,30 @@ +/* + * crypto_stat.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_STAT_H +#define __CRYPTO_STAT_H + +#include + +void crypto_stat_complete_inc(struct crypto_session *s); +void crypto_stat_ptime_inc(struct crypto_session *s); + +#endif /* __CRYPTO_STAT_H */ diff --git a/acrypto/crypto_user.c b/acrypto/crypto_user.c new file mode 100644 index 0000000..ab9b80e --- /dev/null +++ b/acrypto/crypto_user.c @@ -0,0 +1,196 @@ +/* + * crypto_user.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_user.h" + +static struct scatterlist *crypto_user_alloc_sg(int sg_num) +{ + struct scatterlist *sg; + + sg = kmalloc(sg_num * sizeof(*sg), GFP_ATOMIC); + if (!sg) { + dprintk("Failed to allocate %d scatterlist structures.\n", sg_num); + return NULL; + } + + memset(sg, 0, sizeof(*sg) * sg_num); + + return sg; +} + +static void crypto_user_free_sg(struct scatterlist *sg) +{ + kfree(sg); +} + +int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size) +{ + int sg_num; + + if ( (src_size > MAX_DATA_SIZE * PAGE_SIZE) || + (dst_size > MAX_DATA_SIZE * PAGE_SIZE) || + (key_size > MAX_DATA_SIZE * PAGE_SIZE) || + (iv_size > MAX_DATA_SIZE * PAGE_SIZE)) { + dprintk("Sizes are too big: src=%u, dst=%u, key=%u, iv=%u, max=%u.\n", + src_size, dst_size, key_size, iv_size, MAX_DATA_SIZE); + return -EINVAL; + + } + + sg_num = ALIGN_DATA_SIZE(src_size) / PAGE_SIZE; + data->sg_src = crypto_user_alloc_sg(sg_num); + if (!data->sg_src) + goto err_out_exit; + data->sg_src_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(dst_size) / PAGE_SIZE; + data->sg_dst = crypto_user_alloc_sg(sg_num); + if (!data->sg_dst) + goto err_out_free_src; + data->sg_dst_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(key_size) / PAGE_SIZE; + data->sg_key = crypto_user_alloc_sg(sg_num); + if (!data->sg_key) + goto err_out_free_dst; + data->sg_key_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(iv_size) / PAGE_SIZE; + data->sg_iv = crypto_user_alloc_sg(sg_num); + if (!data->sg_iv) + goto err_out_free_key; + data->sg_iv_num = sg_num; + + return 0; + +err_out_free_key: + crypto_user_free_sg(data->sg_key); +err_out_free_dst: + crypto_user_free_sg(data->sg_dst); +err_out_free_src: + crypto_user_free_sg(data->sg_src); +err_out_exit: + + return -ENOMEM; +} + +void crypto_user_free_crypto_data(struct crypto_data *data) +{ + crypto_user_free_sg(data->sg_src); + crypto_user_free_sg(data->sg_dst); + crypto_user_free_sg(data->sg_key); + crypto_user_free_sg(data->sg_iv); +} + +void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg) +{ + int i, sg_num; + + sg_num = ALIGN_DATA_SIZE(size) / PAGE_SIZE; + + dprintk("Filling %d sgs, total size %u: ", sg_num, size); + + for (i=0; i size) + sg[i].length = size; + } else { + sg[i].offset = 0; + sg[i].length = (i != sg_num-1)?PAGE_SIZE:size; + } + dprintka("%x.%x.%p.%lx ", sg[i].offset, sg[i].length, ptr, ALIGN_DATA_SIZE((unsigned long)ptr)); + + size -= sg[i].length; + ptr += sg[i].length; + } + dprintka("\n"); +} + +struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data) +{ + struct scatterlist *sg = NULL; + int inval = 0; + + switch (ud->data_type) { + case CRYPTO_USER_DATA_SRC: + sg = data->sg_src; + inval = (data->sg_src_num * PAGE_SIZE < ud->data_size); + dprintk("Found SRC data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_DST: + sg = data->sg_dst; + inval = (data->sg_dst_num * PAGE_SIZE < ud->data_size); + dprintk("Found DST data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_KEY: + sg = data->sg_key; + inval = (data->sg_key_num * PAGE_SIZE < ud->data_size); + dprintk("Found KEY data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_IV: + sg = data->sg_iv; + inval = (data->sg_iv_num * PAGE_SIZE < ud->data_size); + dprintk("Found IV data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + default: + dprintk("Unknown data type 0x%x, size=%u.\n", ud->data_type, ud->data_size); + break; + } + + return (inval)?NULL:sg; +} + +int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr) +{ + struct scatterlist *sg; + + sg = crypto_user_get_sg(ud, data); + if (!sg) + return -EINVAL; + + crypto_user_fill_sg(ptr, ud->data_size, sg); + + return 0; +} + +EXPORT_SYMBOL_GPL(crypto_user_alloc_crypto_data); +EXPORT_SYMBOL_GPL(crypto_user_free_crypto_data); +EXPORT_SYMBOL_GPL(crypto_user_fill_sg); +EXPORT_SYMBOL_GPL(crypto_user_fill_sg_data); +EXPORT_SYMBOL_GPL(crypto_user_get_sg); diff --git a/acrypto/crypto_user.h b/acrypto/crypto_user.h new file mode 100644 index 0000000..c08cde7 --- /dev/null +++ b/acrypto/crypto_user.h @@ -0,0 +1,52 @@ +/* + * crypto_user.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_H +#define __CRYPTO_USER_H + +#define MAX_DATA_SIZE 3 +#define ALIGN_DATA_SIZE(size) ((size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) + +enum crypto_user_data_types +{ + CRYPTO_USER_DATA_SRC = 0, + CRYPTO_USER_DATA_DST, + CRYPTO_USER_DATA_KEY, + CRYPTO_USER_DATA_IV, +}; + +struct crypto_user_data +{ + __u16 data_size; + __u16 data_type; +}; + + +#ifdef __KERNEL__ + +int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size); +void crypto_user_free_crypto_data(struct crypto_data *data); +void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg); +struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data); +int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_USER_H */ diff --git a/acrypto/crypto_user_direct.c b/acrypto/crypto_user_direct.c new file mode 100644 index 0000000..a9847ef --- /dev/null +++ b/acrypto/crypto_user_direct.c @@ -0,0 +1,391 @@ +/* + * crypto_user_direct.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "crypto_user.h" +#include "crypto_user_direct.h" + +extern struct cb_id crypto_conn_id; + +static LIST_HEAD(crypto_user_direct_list); +static spinlock_t crypto_user_direct_lock = SPIN_LOCK_UNLOCKED; +static struct completion thread_exited; +static int need_exit; +static DECLARE_WAIT_QUEUE_HEAD(crypto_user_direct_wait_queue); +static int crypto_user_direct_has_work; + +static int crypto_user_direct_alloc_pages(struct crypto_user_direct_kern *k) +{ + k->sp = kmalloc(sizeof(struct page *) * k->snum, GFP_KERNEL); + if (!k->sp) { + dprintk("Failed to allocate %d source pages.\n", k->snum); + return -ENOMEM; + } + + k->dp = kmalloc(sizeof(struct page *) * k->dnum, GFP_KERNEL); + if (!k->dp) { + dprintk("Failed to allocate %d destination pages.\n", k->dnum); + kfree(k->sp); + return -ENOMEM; + } + + return 0; +} + +static void crypto_user_direct_free_pages(struct crypto_user_direct_kern *k) +{ + kfree(k->sp); + kfree(k->dp); +} + +static int crypto_user_direct_alloc_vmas(struct crypto_user_direct_kern *k) +{ + k->svma = kmalloc(sizeof(struct vm_area_struct *) * k->snum, GFP_KERNEL); + if (!k->svma) { + dprintk("Failed to allocate %d source vmas.\n", k->snum); + return -ENOMEM; + } + + k->dvma = kmalloc(sizeof(struct vm_area_struct *) * k->dnum, GFP_KERNEL); + if (!k->dvma) { + dprintk("Failed to allocate %d destination vmas.\n", k->dnum); + kfree(k->svma); + return -ENOMEM; + } + + return 0; +} + +static void crypto_user_direct_free_vmas(struct crypto_user_direct_kern *k) +{ + kfree(k->svma); + kfree(k->dvma); +} + +static int crypto_user_direct_alloc_mm(struct crypto_user_direct_kern *k) +{ + int err; + + err = crypto_user_direct_alloc_pages(k); + if (err) + return err; + + err = crypto_user_direct_alloc_vmas(k); + if (err) { + crypto_user_direct_free_pages(k); + return err; + } + + return 0; +} + +static void crypto_user_direct_free_mm(struct crypto_user_direct_kern *k) +{ + crypto_user_direct_free_pages(k); + crypto_user_direct_free_vmas(k); +} + +static void crypto_user_direct_free_data(struct crypto_user_direct_kern *k) +{ + int i; + + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); + for (i=0; idnum; ++i) { + set_page_dirty_lock(k->dp[i]); + page_cache_release(k->dp[i]); + } + up_read(&k->mm->mmap_sem); + crypto_user_direct_free_mm(k); + mmput(k->mm); +} + +static void crypto_user_direct_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_user_direct_kern *k = data->priv; + struct cn_msg m; + + memset(&m, 0, sizeof(m)); + + memcpy(&m.id, &crypto_conn_id, sizeof(m.id)); + m.seq = k->seq; + m.ack = k->ack+1; + + cn_netlink_send(&m, 0, GFP_KERNEL); + + crypto_user_direct_free_data(k); + crypto_user_free_crypto_data(data); +} + +static void crypto_user_direct_fill_data(struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int i, size; + + size = k->usr.src_size; + for (i=0; isg_src_num; ++i) { + data->sg_src[i].page = k->sp[i]; + + if (i == 0) { + data->sg_src[i].offset = offset_in_page(k->usr.src); + data->sg_src[i].length = ALIGN_DATA_SIZE(k->usr.src) - k->usr.src; + } else { + data->sg_src[i].offset = 0; + data->sg_src[i].length = (i != data->sg_src_num)?PAGE_SIZE:size; + } + + size -= data->sg_src[i].length; + } + + size = k->usr.dst_size; + for (i=0; isg_dst_num; ++i) { + data->sg_dst[i].page = k->dp[i]; + + if (i == 0) { + data->sg_dst[i].offset = offset_in_page(k->usr.dst); + data->sg_dst[i].length = ALIGN_DATA_SIZE(k->usr.dst) - k->usr.dst; + } else { + data->sg_dst[i].offset = 0; + data->sg_dst[i].length = (i != data->sg_dst_num)?PAGE_SIZE:size; + } + + size -= data->sg_dst[i].length; + } +} + +static int crypto_user_direct_prepare_data(struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int err, i; + struct task_struct *tsk; + + tsk = find_task_by_pid(k->usr.pid); + if (!tsk) { + dprintk(KERN_ERR "Task with pid=%d does not exist.\n", k->usr.pid); + return -ENODEV; + } + + dprintk("Found task with pid=%d.\n", k->usr.pid); + + k->mm = get_task_mm(tsk); + if (!k->mm) + return -EINVAL; + + k->snum = data->sg_src_num; + k->dnum = data->sg_dst_num; + + err = crypto_user_direct_alloc_mm(k); + if (err) + goto err_out_put_mm; + + down_read(&k->mm->mmap_sem); + + err = get_user_pages(tsk, k->mm, k->usr.src, k->snum, 1, 1, k->sp, k->svma); + if (err < 0) { + dprintk("Failed to get %d src pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_up_sem; + } + + err = get_user_pages(tsk, k->mm, k->usr.dst, k->dnum, 1, 1, k->dp, k->dvma); + if (err < 0) { + dprintk("Failed to get %d dst pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_put_src; + } + + crypto_user_direct_fill_data(data, k); + + return 0; + +err_out_put_src: + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); +err_out_up_sem: + up_read(&k->mm->mmap_sem); + + crypto_user_direct_free_mm(k); +err_out_put_mm: + mmput(k->mm); + return err; +} + +static int crypto_user_direct_prepare(struct crypto_session_initializer *ci, struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int err; + + ci->operation = k->usr.operation; + ci->type = k->usr.type; + ci->mode = k->usr.mode; + ci->priority = k->usr.priority; + ci->callback = crypto_user_direct_callback; + + err = crypto_user_alloc_crypto_data(data, k->usr.src_size, k->usr.dst_size, k->usr.key_size, k->usr.iv_size); + if (err) + return err; + + if (k->usr.key_size) + crypto_user_fill_sg(k->key, k->usr.key_size, data->sg_key); + + if (k->usr.iv_size) + crypto_user_fill_sg(k->iv, k->usr.iv_size, data->sg_iv); + + data->priv = k; + data->priv_size = 0; + + err = crypto_user_direct_prepare_data(data, k); + if (err) { + crypto_user_free_crypto_data(data); + return err; + } + + return 0; +} + +static int crypto_user_direct_process(struct crypto_user_direct_kern *k) +{ + struct crypto_session_initializer ci; + struct crypto_data data; + struct crypto_session *s; + int err; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + err = crypto_user_direct_prepare(&ci, &data, k); + if (err) + return err; + + s = crypto_session_alloc(&ci, &data); + if (!s) { + crypto_user_direct_free_data(k); + return -EINVAL; + } + + return 0; +} + +static int crypto_user_direct_thread(void *data) +{ + struct crypto_user_direct_kern *k; + + daemonize("crypto_user_direct_thread"); + allow_signal(SIGTERM); + + + while (!need_exit) { + wait_event_interruptible_timeout(crypto_user_direct_wait_queue, + crypto_user_direct_has_work, 1000); + crypto_user_direct_has_work = 0; + + spin_lock_bh(&crypto_user_direct_lock); + while(!list_empty(&crypto_user_direct_list)) { + k = list_entry(crypto_user_direct_list.prev, + struct crypto_user_direct_kern, entry); + list_del(&k->entry); + spin_unlock_bh(&crypto_user_direct_lock); + + crypto_user_direct_process(k); + + spin_lock_bh(&crypto_user_direct_lock); + } + spin_unlock_bh(&crypto_user_direct_lock); + } + + complete_and_exit(&thread_exited, 0); +} + +int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr) +{ + struct crypto_user_direct_kern *k; + + k = kmalloc(sizeof(struct crypto_user_direct_kern) + usr->key_size + usr->iv_size, GFP_ATOMIC); + if (!k) { + dprintk("Failed to allocate new kernel crypto request.\n"); + return -ENOMEM; + } + + memset(k, 0, sizeof(*k)); + + memcpy(&k->usr, usr, sizeof(k->usr)); + + k->key = (u8 *)(k+1); + k->iv = (u8 *)(k->key + k->usr.key_size); + + memcpy(k->key, usr->data, k->usr.key_size); + memcpy(k->iv, usr->data + k->usr.key_size, k->usr.iv_size); + + k->seq = seq; + k->ack = ack; + + spin_lock_bh(&crypto_user_direct_lock); + list_add_tail(&k->entry, &crypto_user_direct_list); + spin_unlock_bh(&crypto_user_direct_lock); + + dprintk("msg [%08x.%08x]: op=[%04x.%04x.%04x.%04x], src=%llx [%d], dst=%llx [%d], key=%p [%d], iv=%p [%d].\n", + seq, ack, + k->usr.operation, k->usr.mode, k->usr.type, k->usr.priority, + k->usr.src, k->usr.src_size, + k->usr.dst, k->usr.dst_size, + k->key, k->usr.key_size, + k->iv, k->usr.iv_size); + + crypto_user_direct_has_work = 1; + wake_up_interruptible(&crypto_user_direct_wait_queue); + + return 0; +} + +int crypto_user_direct_init(void) +{ + long pid; + + init_completion(&thread_exited); + pid = kernel_thread(crypto_user_direct_thread, NULL, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + dprintk(KERN_ERR "Failed to create acrypto userspace processing thread.\n"); + return -EINVAL; + } + + printk(KERN_INFO "Acrypto userspace processing thread has been started.\n"); + + return 0; +} + +void crypto_user_direct_fini(void) +{ + need_exit = 1; + wait_for_completion(&thread_exited); + + printk(KERN_INFO "Acrypto userspace processing thread has been finished.\n"); +} diff --git a/acrypto/crypto_user_direct.h b/acrypto/crypto_user_direct.h new file mode 100644 index 0000000..2c94b0c --- /dev/null +++ b/acrypto/crypto_user_direct.h @@ -0,0 +1,74 @@ +/* + * crypto_user_direct.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_DIRECT_H +#define __CRYPTO_USER_DIRECT_H + +struct crypto_user_direct +{ + __u64 src; + __u32 src_size; + __u64 dst; + __u32 dst_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; + + int pid; + + int key_size; + int iv_size; + + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +#include +#include +#include + +struct crypto_user_direct_kern +{ + struct list_head entry; + + u32 seq; + u32 ack; + + struct crypto_user_direct usr; + u8 *key; + u8 *iv; + + int snum, dnum; + struct page **sp, **dp; + struct vm_area_struct **svma, **dvma; + struct mm_struct *mm; +}; + +int crypto_user_direct_init(void); +void crypto_user_direct_fini(void); +int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr); + +#endif /* __KERNEL__ */ + +#endif /* __CRYPTO_USER_DIRECT_H */ diff --git a/acrypto/crypto_user_ioctl.c b/acrypto/crypto_user_ioctl.c new file mode 100644 index 0000000..9a86a94 --- /dev/null +++ b/acrypto/crypto_user_ioctl.c @@ -0,0 +1,263 @@ +/* + * crypto_user_ioctl.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "crypto_user.h" +#include "crypto_user_ioctl.h" + +static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg); +static int crypto_user_ioctl_open(struct inode *inode, struct file *fp); +static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp); +int crypto_user_ioctl_init(void); +void crypto_user_ioctl_fini(void); + +static int crypto_user_ioctl_major = 0; +static char crypto_user_ioctl_name[] = "crypto_user_ioctl"; + +static struct file_operations crypto_user_ioctl_ops = { + .open = crypto_user_ioctl_open, + .release = crypto_user_ioctl_release, + .ioctl = crypto_user_ioctl_ioctl, + .owner = THIS_MODULE +}; + +static void dump_data(u8 *ptr) +{ + int i; + + dprintk("USER DATA: "); + for (i=0; i<32; ++i) + dprintka("%02x ", ptr[i]); + dprintka("\n"); +} + +static int crypto_user_ioctl_open(struct inode *inode, struct file *fp) +{ + struct crypto_user_ioctl_kern *iok; + + iok = kmalloc(sizeof(*iok), GFP_KERNEL); + if (!iok) { + dprintk("Failed to allocate new crypto_user_ioctl_kern structure.\n"); + return -ENOMEM; + } + memset(iok, 0, sizeof(*iok)); + + fp->private_data = iok; + + return 0; +} + +static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp) +{ + struct crypto_user_ioctl_kern *iok = fp->private_data; + int i; + + for (i=0; i<4; ++i) + if (iok->ptr[i]) + kfree(iok->ptr[i]); + kfree(iok); + + return 0; +} + +static void crypto_user_ioctl_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_user_ioctl_kern *iok = data->priv; + + dprintk("%s() for session %08x [%08x].\n", + __func__, iok->s->ci.id, iok->s->ci.dev_id); + + crypto_user_free_crypto_data(&iok->data); + + iok->scompleted = 1; + wake_up_interruptible(&iok->wait); +} + +static int crypto_user_ioctl_session_alloc(struct crypto_user_ioctl *io, struct crypto_user_ioctl_kern *iok) +{ + int err; + + err = crypto_user_alloc_crypto_data(&iok->data, io->src_size, io->dst_size, io->key_size, io->iv_size); + if (err) + return err; + + iok->ci.operation = io->operation; + iok->ci.type = io->type; + iok->ci.mode = io->mode; + iok->ci.priority = io->priority; + iok->ci.callback = crypto_user_ioctl_callback; + + iok->data.priv = iok; + iok->data.priv_size = 0; + + iok->scompleted = 0; + + init_waitqueue_head(&iok->wait); + + iok->s = crypto_session_create(&iok->ci, &iok->data); + if (!iok->s) { + crypto_user_free_crypto_data(&iok->data); + return -ENODEV; + } + + return 0; +} + +static int crypto_user_ioctl_session_add(struct crypto_user_ioctl_kern *iok) +{ + crypto_session_add(iok->s); + + return 0; +} + +static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg) +{ + struct crypto_user_ioctl io; + struct crypto_user_data data; + unsigned long not_read; + int err; + struct crypto_user_ioctl_kern *iok; + + iok = fp->private_data; + + err = 0; + switch (cmd) { + case CRYPTO_SESSION_ALLOC: + not_read = copy_from_user(&io, (void __user *)arg, sizeof(io)); + if (not_read) { + dprintk("Failed to read crypto_user_ioctl structure from userspace.\n"); + err = -EINVAL; + break; + } + + err = crypto_user_ioctl_session_alloc(&io, iok); + break; + case CRYPTO_FILL_DATA: + not_read = copy_from_user(&data, (void __user *)arg, sizeof(data)); + if (not_read) { + dprintk("Failed to read crypto_user_ioctl_data structure from userspace.\n"); + err = -EINVAL; + break; + } + + if (data.data_size > MAX_DATA_SIZE * PAGE_SIZE) { + dprintk("Data size is too bit: size=%u, type=%x.\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (!crypto_user_get_sg(&data, &iok->data)) { + dprintk("Invalid crypto_user_data structure [size=%u, type=%x].\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (iok->ptr[data.data_type]) + kfree(iok->ptr[data.data_type]); + + iok->ptr[data.data_type] = kmalloc(data.data_size, GFP_KERNEL); + if (!iok->ptr[data.data_type]) { + dprintk("Failed to allocate %d bytes for data type %d.\n", + data.data_size, data.data_type); + err = -ENOMEM; + break; + } + + not_read = copy_from_user(iok->ptr[data.data_type], (void __user *)arg + sizeof(data), data.data_size); + if (not_read) { + dprintk("Failed to read %d bytes of crypto data [type=%d] from userspace.\n", + data.data_size, data.data_type); + kfree(iok->ptr[data.data_type]); + err = -EINVAL; + break; + } + + memcpy(&iok->usr[data.data_type], &data, sizeof(struct crypto_user_data)); + + err = crypto_user_fill_sg_data(&data, &iok->data, iok->ptr[data.data_type]); + if (err) { + kfree(iok->ptr[data.data_type]); + break; + } + break; + case CRYPTO_SESSION_ADD: + if (!iok->s) { + dprintk("CRYPTO_SESSION_ADD must be called after session initialisation.\n"); + err = -EINVAL; + break; + } + + err = crypto_user_ioctl_session_add(iok); + if (err) + break; + + wait_event_interruptible(iok->wait, iok->scompleted); + + dump_data(iok->ptr[CRYPTO_USER_DATA_DST]); + + not_read = copy_to_user((void __user *)arg, iok->ptr[CRYPTO_USER_DATA_DST], iok->usr[CRYPTO_USER_DATA_DST].data_size); + if (not_read) { + dprintk("Failed to copy to user %d bytes of result.\n", iok->usr[CRYPTO_USER_DATA_DST].data_size); + err = -EINVAL; + break; + } + break; + + default: + dprintk("Invalid ioctl(0x%x).\n", cmd); + err = -ENODEV; + break; + } + + return err; +} + +int crypto_user_ioctl_init(void) +{ + crypto_user_ioctl_major = register_chrdev(0, crypto_user_ioctl_name, &crypto_user_ioctl_ops); + if (crypto_user_ioctl_major < 0) { + dprintk("Failed to register %s char device: err=%d.\n", crypto_user_ioctl_name, crypto_user_ioctl_major); + return -ENODEV; + }; + + printk("Asynchronous crypto userspace helper(ioctl based) has been started, major=%d.\n", crypto_user_ioctl_major); + + return 0; +} + +void crypto_user_ioctl_fini(void) +{ + unregister_chrdev(crypto_user_ioctl_major, crypto_user_ioctl_name); +} diff --git a/acrypto/crypto_user_ioctl.h b/acrypto/crypto_user_ioctl.h new file mode 100644 index 0000000..b2dfcd0 --- /dev/null +++ b/acrypto/crypto_user_ioctl.h @@ -0,0 +1,67 @@ +/* + * crypto_user_ioctl.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_IOCTL_H +#define __CRYPTO_USER_IOCTL_H + +struct crypto_user_ioctl +{ + __u16 src_size; + __u16 dst_size; + __u16 key_size; + __u16 iv_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; +}; + +#define CRYPTO_USER_IOCTL_SYM 'U' +#define CRYPTO_SESSION_ALLOC _IOW(CRYPTO_USER_IOCTL_SYM, 0, struct crypto_user_ioctl) +#define CRYPTO_SESSION_ADD _IOR(CRYPTO_USER_IOCTL_SYM, 1, char *) +#define CRYPTO_FILL_DATA _IOW(CRYPTO_USER_IOCTL_SYM, 2, struct crypto_user_data) + + +#ifdef __KERNEL__ + +#include + +#include "crypto_user.h" + +struct crypto_user_ioctl_kern +{ + struct crypto_session_initializer ci; + struct crypto_data data; + struct crypto_session *s; + + int scompleted; + wait_queue_head_t wait; + + struct crypto_user_data usr[4]; + void *ptr[4]; +}; + +int crypto_user_ioctl_init(void); +void crypto_user_ioctl_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_USER_IOCTL_H */ diff --git a/acrypto/simple_lb.c b/acrypto/simple_lb.c new file mode 100644 index 0000000..e08d778 --- /dev/null +++ b/acrypto/simple_lb.c @@ -0,0 +1,70 @@ +/* + * simple_lb.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include + +#include "crypto_lb.h" + +extern struct list_head cdev_list; +extern spinlock_t cdev_lock; + +static struct crypto_device *simple_lb_find_device(struct crypto_lb *, + struct crypto_session_initializer *, + struct crypto_data *); + +struct crypto_lb simple_lb = { + .name = "simple_lb", + .find_device = simple_lb_find_device +}; + +static struct crypto_device *simple_lb_find_device(struct crypto_lb *lb, + struct crypto_session_initializer *ci, + struct crypto_data *data) +{ + struct crypto_device *dev, *__dev = NULL; + int min = 0x7ffffff, ref; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (device_broken(dev)) + continue; + if (!match_initializer(dev, ci)) + continue; + + ref = atomic_read(&dev->refcnt); + if (ref < min) { + min = ref; + __dev = dev; + } + } + if (__dev) + crypto_device_get(__dev); + spin_unlock_irqrestore(&cdev_lock, flags); + + return __dev; +} + diff --git a/arch/i386/Kconfig b/arch/i386/Kconfig index 5b1a7d4..c3d7af4 100644 --- a/arch/i386/Kconfig +++ b/arch/i386/Kconfig @@ -1075,6 +1075,8 @@ source "security/Kconfig" source "crypto/Kconfig" +source "acrypto/Kconfig" + source "lib/Kconfig" # diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index e18eb79..bd50058 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -592,4 +592,6 @@ source "security/Kconfig" source "crypto/Kconfig" +source "acrypto/Kconfig" + source "lib/Kconfig" diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index e7a650f..eac4377 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -22,6 +22,24 @@ #define PFX "crypt: " +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE +#include +#include + +struct dm_async_io +{ + atomic_t pending; + int write; + struct completion complete; +}; + +struct crypt_io; +struct dm_async_priv +{ + struct crypt_io *io; +}; +#endif + /* * per bio private data */ @@ -32,6 +50,9 @@ struct crypt_io { struct work_struct work; atomic_t pending; int error; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + struct dm_async_io aio; +#endif }; /* @@ -81,7 +102,9 @@ struct crypt_config { void *iv_gen_private; sector_t iv_offset; unsigned int iv_size; - +#if defined(CONFIG_ACRYPTO) || defined (CONFIG_ACRYPTO_MODULE) + struct crypto_context *ctx; +#endif struct crypto_tfm *tfm; unsigned int key_size; u8 key[0]; @@ -227,11 +250,195 @@ static struct crypt_iv_operations crypt_ .generator = crypt_iv_essiv_gen }; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + +static mempool_t *dm_acrypto_pool; +static kmem_cache_t *dm_acrypto_cache; +static int dm_acrypto_size = sizeof(struct dm_async_priv) + 4*sizeof(struct scatterlist) + 128*2; +static int dm_acrypto_use = 1; +module_param(dm_acrypto_use, uint, 0); + +static inline u16 crypto_tfm_get_type(struct crypto_tfm *tfm, int key_size) +{ + u16 type; + char *name; + + name = (char *)crypto_tfm_alg_name(tfm); + + if (!strncmp(name, "aes", 3)) { + switch (key_size << 3) { + case 128: + type = CRYPTO_TYPE_AES_128; + break; + case 192: + type = CRYPTO_TYPE_AES_192; + break; + case 256: + type = CRYPTO_TYPE_AES_256; + break; + default: + type = 0xffff; + break; + } + } else if (!strncmp(name, "des3_ede", 3)) + type = CRYPTO_TYPE_3DES; + else + type = 0xffff; + + return type; +} + +static inline u16 crypto_tfm_get_mode(struct crypto_tfm *tfm) +{ + u16 mode = tfm->crt_cipher.cit_mode & CRYPTO_TFM_MODE_MASK; + + switch (mode) { + case CRYPTO_TFM_MODE_ECB: + mode = CRYPTO_MODE_ECB; + break; + case CRYPTO_TFM_MODE_CBC: + mode = CRYPTO_MODE_CBC; + break; + case CRYPTO_TFM_MODE_CFB: + mode = CRYPTO_MODE_CFB; + break; + case CRYPTO_TFM_MODE_CTR: + mode = CRYPTO_MODE_CTR; + break; + default: + mode = 0xffff; + break; + } + + return mode; +} + +static void dec_pending(struct crypt_io *io, int error); + +static inline void dm_finish(struct crypt_io *io, int error) +{ + if (atomic_dec_and_test(&io->aio.pending)) { + if (io->aio.write) + complete(&io->aio.complete); + else + dec_pending(io, error); + } +} + +static void dm_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct dm_async_priv *priv = data->priv; + + BUG_ON(!priv); + BUG_ON(!priv->io); + + dm_finish(priv->io, 0); + crypto_context_put(data->ctx); + + mempool_free(priv, dm_acrypto_pool); +} + +static inline int acrypto_process(struct crypt_config *cc, struct crypt_io *io, struct scatterlist *out, + struct scatterlist *in, unsigned int len, u8 *iv, int iv_size, int pos) +{ + struct crypto_session *s; + struct crypto_session_initializer ci; + struct crypto_data data; + struct scatterlist *sg; + int err = 0; + u8 *local_key; + struct dm_async_priv *priv; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + ci.operation = (io->aio.write)?CRYPTO_OP_ENCRYPT:CRYPTO_OP_DECRYPT; + ci.priority = 0; + ci.callback = &dm_callback; + ci.type = crypto_tfm_get_type(cc->tfm, cc->key_size); + ci.mode = crypto_tfm_get_mode(cc->tfm); + + if (ci.type == 0xffff || ci.mode == 0xffff) { + err = -EINVAL; + goto err_out_exit; + } + + data.sg_src_num = data.sg_dst_num = data.sg_key_num = 1; + data.sg_iv_num = (iv)?1:0; + + priv = mempool_alloc(dm_acrypto_pool, GFP_KERNEL); + if (!priv) { + err = -ENOMEM; + goto err_out_exit; + } + + priv->io = io; + + data.priv = priv; + + sg = (struct scatterlist *)(priv + 1); + + data.sg_src = &sg[0]; + data.sg_dst = &sg[1]; + + data.sg_src[0].page = in->page; + data.sg_src[0].offset = in->offset; + data.sg_src[0].length = in->length; + + data.sg_dst[0].page = out->page; + data.sg_dst[0].offset = out->offset; + data.sg_dst[0].length = out->length; + + data.sg_key = &sg[2]; + local_key = (u8 *)&sg[3]; + + memcpy(local_key, cc->key, cc->key_size); + + data.sg_key[0].page = virt_to_page(local_key); + data.sg_key[0].offset = offset_in_page(local_key); + data.sg_key[0].length = cc->key_size; + + if (iv) { + u8 *local_iv; + + data.sg_iv = (struct scatterlist *)(local_key + cc->key_size); + local_iv = (u8 *)(local_key + cc->key_size + sizeof(*sg)); + + data.sg_iv[0].page = virt_to_page(local_iv); + data.sg_iv[0].offset = offset_in_page(local_iv); + data.sg_iv[0].length = iv_size; + + memcpy(local_iv, iv, iv_size); + } + + data.ctx = crypto_context_get(cc->ctx); + s = crypto_session_alloc(&ci, &data); + if (!s) { + err = -ENOMEM; + goto err_out_free; + } + + return 0; + +err_out_free: + crypto_context_put(cc->ctx); + mempool_free(priv, dm_acrypto_pool); +err_out_exit: + + return err; +} +#else +static inline int acrypto_process(struct crypt_config *cc, struct crypt_io *io, struct scatterlist *out, + struct scatterlist *in, unsigned int len, u8 *iv, int iv_size, int write, int pos) +{ + return 0; +} +#endif static int crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out, struct scatterlist *in, unsigned int length, - int write, sector_t sector) + int write, sector_t sector, struct crypt_io *io, int pos) { u8 iv[cc->iv_size]; int r; @@ -241,11 +448,19 @@ crypt_convert_scatterlist(struct crypt_c if (r < 0) return r; + r = acrypto_process(cc, io, out, in, length, iv, cc->iv_size, pos); + if (r == 0) + return r; + if (write) r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv); else r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv); } else { + r = acrypto_process(cc, io, out, in, length, NULL, 0, pos); + if (r == 0) + return r; + if (write) r = crypto_cipher_encrypt(cc->tfm, out, in, length); else @@ -274,9 +489,50 @@ crypt_convert_init(struct crypt_config * * Encrypt / decrypt data from one bio to another one (can be the same one) */ static int crypt_convert(struct crypt_config *cc, - struct convert_context *ctx) + struct convert_context *ctx, + struct crypt_io *io) { int r = 0; + sector_t first_sector = ctx->sector; + +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + int num = 0; + + if (dm_acrypto_use) { + unsigned int idx_in, idx_out, offset_in, offset_out; + + idx_in = ctx->idx_in; + idx_out = ctx->idx_out; + offset_in = ctx->offset_in; + offset_out = ctx->offset_out; + + while(idx_in < ctx->bio_in->bi_vcnt && + idx_out < ctx->bio_out->bi_vcnt) { + + struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, idx_in); + struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, idx_out); + + offset_in += 1 << SECTOR_SHIFT; + offset_out += 1 << SECTOR_SHIFT; + + if (offset_in >= bv_in->bv_len) { + offset_in = 0; + idx_in++; + } + + if (offset_out >= bv_out->bv_len) { + offset_out = 0; + idx_out++; + } + + num++; + } + + init_completion(&io->aio.complete); + atomic_set(&io->aio.pending, num); + io->aio.write = ctx->write; + } +#endif while(ctx->idx_in < ctx->bio_in->bi_vcnt && ctx->idx_out < ctx->bio_out->bi_vcnt) { @@ -306,13 +562,26 @@ static int crypt_convert(struct crypt_co } r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length, - ctx->write, ctx->sector); + ctx->write, ctx->sector, io, + ctx->sector - first_sector); if (r < 0) break; - ctx->sector++; } + /* + * If there is an IO here, then this request was done through crypt_endio() + * and queued work and we can simply return from this request. + * If there is no IO here, then it was bulk of requests from crypt_map() + * and we can not just return, since crypt_map() is designed + * to be synchronous. + */ + if (!ctx->write && r) + dec_pending(io, r); + + if (ctx->write) + wait_for_completion(&io->aio.complete); + return r; } @@ -461,13 +730,10 @@ static void kcryptd_do_work(void *data) struct crypt_io *io = (struct crypt_io *) data; struct crypt_config *cc = (struct crypt_config *) io->target->private; struct convert_context ctx; - int r; crypt_convert_init(cc, &ctx, io->bio, io->bio, io->bio->bi_sector - io->target->begin, 0); - r = crypt_convert(cc, &ctx); - - dec_pending(io, r); + crypt_convert(cc, &ctx, io); } static void kcryptd_queue_io(struct crypt_io *io) @@ -593,6 +859,13 @@ static int crypt_ctr(struct dm_target *t ti->error = PFX "Expected cipher algorithm"; goto bad2; } +#if defined(CONFIG_ACRYPTO) || defined (CONFIG_ACRYPTO_MODULE) + cc->ctx = crypto_context_alloc(crypto_tfm_get_mode(tfm), + crypto_tfm_get_type(tfm, key_size), + cc->key, key_size); + if (!cc->ctx) + goto bad2; +#endif cc->tfm = tfm; @@ -648,6 +921,9 @@ static int crypt_ctr(struct dm_target *t ti->error = PFX "Error setting key"; goto bad5; } +#if defined(CONFIG_ACRYPTO) || defined (CONFIG_ACRYPTO_MODULE) + crypto_context_notify(cc->ctx, CRYPTO_DEV_CTX_KEY); +#endif if (sscanf(argv[2], SECTOR_FORMAT, &cc->iv_offset) != 1) { ti->error = PFX "Invalid iv_offset sector"; @@ -688,6 +964,9 @@ bad3: if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) cc->iv_gen_ops->dtr(cc); bad2: +#if defined(CONFIG_ACRYPTO) || defined (CONFIG_ACRYPTO_MODULE) + crypto_context_put(cc->ctx); +#endif crypto_free_tfm(tfm); bad1: /* Must zero key material before freeing */ @@ -757,7 +1036,7 @@ crypt_clone(struct crypt_config *cc, str io->first_clone, bvec_idx); if (clone) { ctx->bio_out = clone; - if (crypt_convert(cc, ctx) < 0) { + if (crypt_convert(cc, ctx, io) < 0) { crypt_free_buffer_pages(cc, clone, clone->bi_size); bio_put(clone); @@ -921,6 +1200,25 @@ static struct target_type crypt_target = static int __init dm_crypt_init(void) { int r; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + printk("%s: Compiled with ACRYPTO support.\n", __func__); + + dm_acrypto_cache = kmem_cache_create("dm_acrypto_io", dm_acrypto_size, 0, 0, NULL, NULL); + if (!dm_acrypto_cache) { + dm_acrypto_use = 0; + goto sync_init; + } + + dm_acrypto_pool = mempool_create(MIN_IOS, mempool_alloc_slab, + mempool_free_slab, dm_acrypto_cache); + if (!dm_acrypto_pool) { + kmem_cache_destroy(dm_acrypto_cache); + dm_acrypto_use = 0; + goto sync_init; + } + +sync_init: +#endif _crypt_io_pool = kmem_cache_create("dm-crypt_io", sizeof(struct crypt_io), @@ -947,6 +1245,12 @@ bad2: destroy_workqueue(_kcryptd_workqueue); bad1: kmem_cache_destroy(_crypt_io_pool); +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + if (dm_acrypto_use) { + mempool_destroy(dm_acrypto_pool); + kmem_cache_destroy(dm_acrypto_cache); + } +#endif return r; } @@ -959,6 +1263,12 @@ static void __exit dm_crypt_exit(void) destroy_workqueue(_kcryptd_workqueue); kmem_cache_destroy(_crypt_io_pool); +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + if (dm_acrypto_use) { + mempool_destroy(dm_acrypto_pool); + kmem_cache_destroy(dm_acrypto_cache); + } +#endif } module_init(dm_crypt_init); diff --git a/include/linux/acrypto.h b/include/linux/acrypto.h new file mode 100644 index 0000000..3447714 --- /dev/null +++ b/include/linux/acrypto.h @@ -0,0 +1,307 @@ +/* + * acrypto.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_H +#define __ACRYPTO_H + +#define SCACHE_NAMELEN 32 + +struct crypto_session_initializer; +struct crypto_data; +typedef void (*crypto_callback_t) (struct crypto_session_initializer *, + struct crypto_data *); + +struct crypto_device_stat +{ + __u64 scompleted; + __u64 sfinished; + __u64 sstarted; + __u64 kmem_failed; + __u64 route_failed; + __u64 pool_failed; +}; + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include +#include +#include + +#include + +//#define DEBUG +#ifdef DEBUG +#define dprintk(f, a...) printk(f, ##a) +#define dprintka(f, a...) printk(f, ##a) +#else +#define dprintk(f, a...) +#define dprintka(f, a...) +#endif + +#define SESSION_COMPLETED 15 +#define SESSION_DIRECT 12 /* Crypto callback is called directly from crypto processing engine */ +#define SESSION_BOUND 11 +#define SESSION_BROKEN 10 +#define SESSION_FROM_CACHE 9 +#define SESSION_DATA_READY_SYNC 8 /* Crypto procesing is being done in ->data_ready() callback */ + +#define session_completed(s) test_bit(SESSION_COMPLETED, &s->ci.flags) +struct crypto_session; +extern int complete_session(struct crypto_session *s); +#define uncomplete_session(s) clear_bit(SESSION_COMPLETED, &s->ci.flags) + +#define session_bound(s) test_bit(SESSION_BOUND, &s->ci.flags) +#define bind_session(s) set_bit(SESSION_BOUND, &s->ci.flags) +#define unbind_session(s) clear_bit(SESSION_BOUND, &s->ci.flags) +#define sci_bound(ci) test_bit(SESSION_BOUND, &ci->flags) + +#define session_broken(s) test_bit(SESSION_BROKEN, &s->ci.flags) +#define break_session(s) set_bit(SESSION_BROKEN, &s->ci.flags) +#define unbreak_session(s) clear_bit(SESSION_BROKEN, &s->ci.flags) + +#define session_from_cache(s) test_bit(SESSION_FROM_CACHE, &s->ci.flags) +#define mark_session_from_cache(s) set_bit(SESSION_FROM_CACHE, &s->ci.flags) + +#define CRYPTO_MAX_PRIV_SIZE 1024 + +#define DEVICE_BROKEN 0 + +#define device_broken(dev) test_bit(DEVICE_BROKEN, &dev->flags) +#define break_device(dev) set_bit(DEVICE_BROKEN, &dev->flags) +#define repair_device(dev) clear_bit(DEVICE_BROKEN, &dev->flags) + +struct crypto_capability { + u16 operation; + u16 type; + u16 mode; + u16 qlen; + u64 ptime; + u64 scomp; +}; + +#define CRYPTO_DEV_CTX_INIT (1<<0) +#define CRYPTO_DEV_CTX_EXIT (1<<1) +#define CRYPTO_DEV_CTX_KEY (1<<2) + +struct crypto_context +{ + struct list_head entry; + u16 type; + u16 mode; + atomic_t refcnt; + u8 *key; + unsigned int key_size; + void *(driver_priv[0]); /* actual size is max_ca_drivers */ +}; + +extern int crypto_context_register(struct crypto_context *ctx); +extern void crypto_context_unregister(struct crypto_context *ctx); +extern struct crypto_context *crypto_context_alloc(u16 type, u16 mode, u8 *key, unsigned int key_size); +extern void crypto_context_free(struct crypto_context *ctx); +extern void crypto_context_notify(struct crypto_context *ctx, u32 flags); + +static inline struct crypto_context *crypto_context_get(struct crypto_context *ctx) +{ + atomic_inc(&ctx->refcnt); + return ctx; +} + +static inline void crypto_context_put(struct crypto_context *ctx) +{ + if (ctx && atomic_dec_and_test(&ctx->refcnt)) + crypto_context_free(ctx); +} + +struct crypto_session_initializer { + u16 operation; + u16 type; + u16 mode; + u16 priority; + + u32 id; + u32 dev_id; + + long flags; + + u32 bdev; + + u64 ptime; + + crypto_callback_t callback; +}; + +struct crypto_data { + struct scatterlist *sg_src; + int sg_src_num; + struct scatterlist *sg_dst; + int sg_dst_num; + struct scatterlist *sg_key; + int sg_key_num; + struct scatterlist *sg_iv; + int sg_iv_num; + + struct crypto_context *ctx; + void *priv; + unsigned int priv_size; +}; + +struct crypto_queue +{ + struct list_head queue_entry; + struct list_head session_list; + unsigned int qlen; + u16 priority; +}; + +struct crypto_queue_bundle +{ + struct list_head queue_list; + spinlock_t queue_lock; + unsigned int num; +}; + +struct crypto_device { + char name[SCACHE_NAMELEN]; + char route_cache_name[SCACHE_NAMELEN]; + char queue_cache_name[SCACHE_NAMELEN]; + + struct crypto_queue_bundle queue; + + atomic_t sid; + spinlock_t lock; + + atomic_t refcnt; + + long flags; + + u32 id; + + struct list_head cdev_entry; + + void (*data_ready)(struct crypto_device *); + + struct device_driver *driver; + struct device device; + struct class_device class_device; + struct completion dev_released; + + spinlock_t stat_lock; + struct crypto_device_stat stat; + + struct crypto_capability *cap; + int cap_number; + + void *priv; + + mempool_t *queue_pool; + kmem_cache_t *queue_cache; + + mempool_t *session_pool; + kmem_cache_t *session_cache; + + mempool_t *route_pool; + kmem_cache_t *route_cache; + + void (*context_callback)(struct crypto_device *dev, + struct crypto_context *ctx, u32 flags); + int ca_index; /* index of context-aware driver */ +}; + +/* DRIVER_PRIV() macro is used by context-aware drivers to access + * per-context-per-driver private pointer. + */ +#define DRIVER_PRIV(dev,ctx) ((ctx)->driver_priv[(dev)->ca_index]) + +struct crypto_route_head { + struct crypto_route *next; + struct crypto_route *prev; + + u32 qlen; + spinlock_t lock; +}; + +struct crypto_route { + struct crypto_route *next; + struct crypto_route *prev; + + struct crypto_route_head *list; + struct crypto_device *dev; + + struct crypto_session_initializer ci; +}; + +struct crypto_session { + struct list_head dev_queue_entry; + + struct crypto_session_initializer ci; + + struct crypto_data data; + + spinlock_t lock; + + struct work_struct work; + + struct crypto_route_head route_list; + + struct crypto_device *pool_dev; +}; + +struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *, struct crypto_data *); +struct crypto_session *crypto_session_create(struct crypto_session_initializer *, struct crypto_data *); +void crypto_session_destroy(struct crypto_session *); +int __crypto_session_add(struct crypto_device *, struct crypto_session *); +int crypto_session_add(struct crypto_session *); + +int crypto_session_dequeue_route(struct crypto_session *); +void crypto_destroy_queues(struct crypto_device *dev); + +void crypto_device_get(struct crypto_device *); +void crypto_device_put(struct crypto_device *); +struct crypto_device *crypto_device_get_name(char *); + +int __crypto_device_add(struct crypto_device *); +int crypto_device_add(struct crypto_device *); +void __crypto_device_remove(struct crypto_device *); +void crypto_device_remove(struct crypto_device *); +int match_initializer(struct crypto_device *, struct crypto_session_initializer *); +int __match_initializer(struct crypto_capability *, struct crypto_session_initializer *, struct crypto_device *); + +int crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s); +struct crypto_session *crypto_session_dequeue(struct crypto_device *dev); + +static inline int compare_device(struct crypto_device *d1, struct crypto_device *d2) +{ + return (d1->id == d2->id)?1:0; +} + +void crypto_process(struct crypto_session *s, + int (*proc)(struct crypto_session *s, void *src, int slen, void *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + int (*proc_scatter)(struct crypto_session *s, struct scatterlist *src, struct scatterlist *dst, int nbytes, + void *key, int klen, void *iv, int ilen, void *data), + void (*finish)(struct crypto_session *s, void *key, int klen, void *iv, int ilen, void *data), + void *data); + +#endif /* __KERNEL__ */ +#endif /* __ACRYPTO_H */ diff --git a/include/linux/crypto_def.h b/include/linux/crypto_def.h new file mode 100644 index 0000000..b701b1e --- /dev/null +++ b/include/linux/crypto_def.h @@ -0,0 +1,41 @@ +/* + * crypto_def.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_DEF_H +#define __CRYPTO_DEF_H + +#define CRYPTO_OP_DECRYPT 0 +#define CRYPTO_OP_ENCRYPT 1 +#define CRYPTO_OP_HMAC 2 +#define CRYPTO_OP_RNG 3 + +#define CRYPTO_MODE_ECB 0 +#define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_CFB 2 +#define CRYPTO_MODE_OFB 3 +#define CRYPTO_MODE_CTR 4 + +#define CRYPT