diff --git a/Makefile b/Makefile index f4218b5..dc0ea0f 100644 --- a/Makefile +++ b/Makefile @@ -584,7 +584,7 @@ export MODLIB ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ acrypto/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/acrypto/Kconfig b/acrypto/Kconfig new file mode 100644 index 0000000..42946f1 --- /dev/null +++ b/acrypto/Kconfig @@ -0,0 +1,29 @@ +menu "Asynchronous crypto layer" + +config ACRYPTO + tristate "Asynchronous crypto layer" + select CONNECTOR + --- help --- + It supports: + - multiple asynchronous crypto device queues + - crypto session routing + - crypto session binding + - modular load balancing + - crypto session batching genetically implemented by design + - crypto session priority + - different kinds of crypto operation(RNG, asymmetrical crypto, HMAC and any other + +config ASYNC_PROVIDER + tristate "Asynchronous crypto provider (AES CBC)" + depends on ACRYPTO && (CRYPTO_AES || CRYPTO_AES_586 || CRYPTO_AES_X86_64) + --- help --- + Asynchronous crypto provider based on synchronous crypto layer. + It supports AES CBC crypto mode (may be changed by source edition). + +config CONSUMER + tristate "Test asynchronous crypto consumer" + depends on ACRYPTO + --- help --- + Test asynchronous crypto consumer. + +endmenu diff --git a/acrypto/Makefile b/acrypto/Makefile new file mode 100644 index 0000000..f3e40dc --- /dev/null +++ b/acrypto/Makefile @@ -0,0 +1,13 @@ +obj-$(CONFIG_ACRYPTO) += acrypto.o +obj-$(CONFIG_ASYNC_PROVIDER) += async_provider.o +obj-$(CONFIG_CONSUMER) += consumer.o + +acrypto-y += crypto_main.o +acrypto-y += crypto_lb.o +acrypto-y += crypto_dev.o +acrypto-y += crypto_conn.o +acrypto-y += crypto_stat.o +acrypto-y += crypto_user_direct.o +acrypto-y += crypto_user_ioctl.o +acrypto-y += crypto_user.o +acrypto-y += simple_lb.o diff --git a/acrypto/async_provider.c b/acrypto/async_provider.c new file mode 100644 index 0000000..7397d6d --- /dev/null +++ b/acrypto/async_provider.c @@ -0,0 +1,265 @@ +/* + * async_provider.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "crypto_route.h" +#include "crypto_user.h" + +static unsigned int trnum = 1; +module_param(trnum, uint, 0); + +static void prov_data_ready(struct crypto_device *); + +static struct crypto_capability prov_caps[] = { + {CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 1000}, + {CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_ECB, 1000}, + + {CRYPTO_OP_ENCRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 1000}, + {CRYPTO_OP_DECRYPT, CRYPTO_TYPE_AES_128, CRYPTO_MODE_CBC, 1000}, +}; +static int prov_cap_number = sizeof(prov_caps)/sizeof(prov_caps[0]); + +static int need_exit; +static char async_algo[] = "aes"; +static char async_key[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +static struct crypto_device pdev = { + .name = "async_provider", + .data_ready = prov_data_ready, + .cap = &prov_caps[0], +}; + +static struct async_provider +{ + int num; + struct crypto_tfm *tfm; + int work; + wait_queue_head_t async_wait_queue; + struct completion thread_exited; + struct crypto_device pdev; + int setup; +} *aprov; + +static void prov_data_ready(struct crypto_device *dev) +{ + struct async_provider *p; + + p = (struct async_provider *)dev->priv; + + if (p) { + p->work = 1; + wake_up_interruptible(&p->async_wait_queue); + } +} + +int async_process(struct crypto_session *s, struct scatterlist *src, int slen, + struct scatterlist *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data) +{ + int err = -EINVAL; + struct async_provider *p = (struct async_provider *)data; + + if (!p->setup && key && klen) { + err = crypto_cipher_setkey(p->tfm, key, klen); + if (err) { + printk("Failed to set key [keylen=%d]: err=%d.\n", klen, err); + return err; + } + + if (s->ci.mode != CRYPTO_MODE_ECB) { + if (!iv || !ilen || (ilen != crypto_tfm_alg_blocksize(p->tfm))) { + printk("Crypto mode %d requires IV, which is broken: iv=%p, ivlen=%d, blocksize=%u.\n", + s->ci.mode, iv, ilen, crypto_tfm_alg_blocksize(p->tfm)); + return -EINVAL; + } + + crypto_cipher_set_iv(p->tfm, iv, ilen); + } + p->setup = 1; + } + + BUG_ON(dst->offset + dlen > PAGE_SIZE); + + if (s->ci.operation == CRYPTO_OP_ENCRYPT) + err = crypto_cipher_encrypt(p->tfm, dst, src, dlen); + else + err = crypto_cipher_decrypt(p->tfm, dst, src, dlen); + + return err; +} + +static void async_finish(struct crypto_session *s, void *data) +{ + struct async_provider *p = (struct async_provider *)data; + + p->setup = 0; + set_bit(SESSION_DIRECT, &s->ci.flags); +} + +static int async_thread(void *data) +{ + struct async_provider *p = (struct async_provider *)data; + struct crypto_device *dev = &p->pdev; + struct crypto_session *s; + + daemonize("%s", dev->name); + allow_signal(SIGTERM); + + while (!need_exit) { + p->work = 0; + while ((s = crypto_session_dequeue(dev)) != NULL) { + crypto_process(s, NULL, &async_process, &async_finish, data); + } + + wait_event_interruptible_timeout(p->async_wait_queue, p->work, HZ); + } + + complete_and_exit(&p->thread_exited, 0); +} + +static int prov_init_one(struct async_provider *p) +{ + long pid; + int err; + + init_waitqueue_head(&p->async_wait_queue); + + p->tfm = crypto_alloc_tfm(async_algo, CRYPTO_TFM_MODE_CBC); + if (!p->tfm) { + printk(KERN_ERR "Failed to allocate %d's %s tfm.\n", p->num, async_algo); + return -EINVAL; + } + + err = crypto_cipher_setkey(p->tfm, async_key, sizeof(async_key)); + if (err) { + printk("Failed to set key [keylen=%zu]: err=%d.\n", + sizeof(async_key), err); + goto err_out_free_tfm; + } + + init_completion(&p->thread_exited); + + memcpy(&p->pdev, &pdev, sizeof(pdev)); + snprintf(p->pdev.name, sizeof(p->pdev.name), "async_provider%d", p->num); + + p->pdev.cap_number = prov_cap_number; + p->pdev.priv = p; + + pid = kernel_thread(async_thread, p, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + err = -EINVAL; + printk(KERN_ERR "Failed to create kernel load balancing thread.\n"); + goto err_out_free_tfm; + } + + err = crypto_device_add(&p->pdev); + if (err) + goto err_out_remove_thread; + + return 0; + +err_out_remove_thread: + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); +err_out_free_tfm: + crypto_free_tfm(p->tfm); + + return err; +} + +static void prov_fini_one(struct async_provider *p) +{ + crypto_device_remove(&p->pdev); + need_exit = 1; + p->work = 1; + wake_up(&p->async_wait_queue); + wait_for_completion(&p->thread_exited); + + crypto_free_tfm(p->tfm); +} + +int prov_init(void) +{ + int err, i; + + aprov = kmalloc(trnum * sizeof(struct async_provider), GFP_KERNEL); + if (!aprov) { + printk(KERN_ERR "Failed to allocate %d async_provider pointers.\n", trnum); + return -ENOMEM; + } + + memset(aprov, 0, trnum * sizeof(struct async_provider)); + + for (i=0; i= 0) + prov_fini_one(&aprov[i]); + + kfree(aprov); + + return err; +} + +void prov_fini(void) +{ + int i; + + for (i=0; i"); +MODULE_DESCRIPTION("Test crypto module provider."); diff --git a/acrypto/consumer.c b/acrypto/consumer.c new file mode 100644 index 0000000..1ac8265 --- /dev/null +++ b/acrypto/consumer.c @@ -0,0 +1,266 @@ +/* + * consumer.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef DEBUG +#include +#include +#include "crypto_user.h" + +#undef dprintk +//#define dprintk(f, a...) printk(f, ##a) +#define dprintk(f, a...) do {} while(0) + +#define KEY_LENGTH 16 +static char ckey[KEY_LENGTH]; +static char civ[KEY_LENGTH]; +static int key_length = sizeof(ckey); +static int iv_length = sizeof(civ); + +static void ctest_callback(struct crypto_session_initializer *ci, + struct crypto_data *data); + +static struct crypto_session_initializer ci = { + .operation = CRYPTO_OP_ENCRYPT, + .type = CRYPTO_TYPE_AES_128, + .mode = CRYPTO_MODE_CBC, + .priority = 4, + .callback = ctest_callback, +}; + +static struct crypto_data cdata; + +#define CSESSION_MAX 1000 +static struct crypto_session *s; +static atomic_t watermark; +static struct timer_list ctimer; +static char data_str[4096] = "test message qwerty"; + +static void ctest_callback(struct crypto_session_initializer *ci, + struct crypto_data *data) +{ + int i, off, size, ssize; + struct scatterlist *sg; + unsigned char *ptr; + + dprintk("%s: session %llu [%llu]\n", __func__, ci->id, ci->dev_id); + dprintk("src=%s, len=%d.\n", + ((char *)page_address(data->sg_src[0].page)) + data->sg_src[0].offset, + data->sg_src[0].length); +#if 1 + ssize = 32; + + sg = &data->sg_key[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("key[%d]=", size); + for (i = 0; i < size; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_src[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("src[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); + + sg = &data->sg_dst[0]; + ptr = (unsigned char *)page_address(sg->page); + off = sg->offset; + size = sg->length; + + dprintk("dst[%d]=", size); + for (i = 0; i < ssize; ++i) + dprintk("0x%02x, ", ptr[i + off]); + dprintk("\n"); +#endif + atomic_dec(&watermark); +} + +int ctimer_func(void *data, int size, int op) +{ + u8 *ptr; + + if (size > PAGE_SIZE) + size = PAGE_SIZE; + + ptr = kmap_atomic(cdata.sg_src[0].page, KM_USER1); + if (!ptr) + return -1; + + memcpy(ptr + cdata.sg_src[0].offset, data, size); + cdata.sg_src[0].length = size; + cdata.sg_dst[0].length = size; + + kunmap_atomic(ptr, KM_USER1); + + ci.operation = op; + s = crypto_session_alloc(&ci, &cdata); + if (s) + atomic_inc(&watermark); + + return (s) ? 0 : -EINVAL; +} + +static int alloc_sg(struct scatterlist *sg, void *data, int size) +{ + sg->offset = 0; + sg->page = alloc_pages(GFP_KERNEL, get_order(size)); + if (!sg->page) { + printk(KERN_ERR "Failed to allocate page.\n"); + return -ENOMEM; + } + + if (data) { + void *ptr = kmap(sg->page); + + if (!ptr) { + __free_pages(sg->page, get_order(size)); + return -ENOMEM; + } + + memcpy(ptr, data, size); + kunmap(ptr); + } + + sg->length = size; + + return 0; +} + +static void ctimerf(unsigned long data) +{ + int size, err, i; + + dprintk("%s started\n", __func__); + + size = 4096; + + for (i=0; i<1000; ++i) + err = ctimer_func(data_str, size, CRYPTO_OP_ENCRYPT); + //err = ctimer_func(data_str, size, CRYPTO_OP_DECRYPT); + + if (!err) + mod_timer(&ctimer, jiffies + 1); + else + mod_timer(&ctimer, jiffies + 1); + + dprintk("%s finished.\n", __func__); +} + +int consumer_init(void) +{ + int err; + + err = crypto_user_alloc_crypto_data(&cdata, sizeof(data_str), sizeof(data_str), key_length, iv_length); + if (err) + return err; + + err = alloc_sg(&cdata.sg_src[0], NULL, PAGE_SIZE); + if (err) + goto err_out_return; + + err = alloc_sg(&cdata.sg_dst[0], NULL, PAGE_SIZE); + if (err) + goto err_out_src; + + err = alloc_sg(&cdata.sg_key[0], ckey, key_length); + if (err) + goto err_out_dst; + + err = alloc_sg(&cdata.sg_iv[0], civ, iv_length); + if (err) + goto err_out_key; + + //cdata.priv_size = 256;//sizeof(struct aes_ctx); + + init_timer(&ctimer); + ctimer.function = &ctimerf; + ctimer.expires = jiffies + HZ; + ctimer.data = 0; + + add_timer(&ctimer); + + return 0; + + __free_pages(cdata.sg_iv[0].page, get_order(1)); +err_out_key: + __free_pages(cdata.sg_key[0].page, get_order(key_length)); +err_out_dst: + __free_pages(cdata.sg_dst[0].page, get_order(1)); +err_out_src: + __free_pages(cdata.sg_src[0].page, get_order(1)); +err_out_return: + crypto_user_free_crypto_data(&cdata); + + return -ENODEV; +} + +void consumer_fini(void) +{ + del_timer_sync(&ctimer); + + printk(KERN_INFO "%s: Timer has been removed.\n", __func__); + while (atomic_read(&watermark)) { + msleep(1000); + + printk(KERN_INFO "Waiting for sessions to be freed: watermark=%d.\n", + atomic_read(&watermark)); + } + + if (!cdata.sg_key[0].length) + printk("BUG: key length is 0 in %s.\n", __func__); + + __free_pages(cdata.sg_iv[0].page, get_order(1)); + __free_pages(cdata.sg_key[0].page, get_order(key_length)); + __free_pages(cdata.sg_dst[0].page, get_order(1)); + __free_pages(cdata.sg_src[0].page, get_order(1)); + + crypto_user_free_crypto_data(&cdata); + + printk(KERN_INFO "Test crypto module consumer is unloaded.\n"); +} + +module_init(consumer_init); +module_exit(consumer_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Test crypto module consumer."); diff --git a/acrypto/crypto_conn.c b/acrypto/crypto_conn.c new file mode 100644 index 0000000..cd4f1d1 --- /dev/null +++ b/acrypto/crypto_conn.c @@ -0,0 +1,135 @@ +/* + * crypto_conn.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" + +#include + +#include "crypto_conn.h" +#include "crypto_user_ioctl.h" +#include "crypto_user_direct.h" + +struct cb_id crypto_conn_id = { 0xdead, 0x0000 }; +static char crypto_conn_name[] = "crconn"; + +static void crypto_conn_callback(void *data) +{ + struct cn_msg *msg, *reply; + struct crypto_conn_data *d, *cmd; + struct crypto_device *dev; + + msg = (struct cn_msg *)data; + d = (struct crypto_conn_data *)msg->data; + + if (msg->len < sizeof(*d)) { + dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u < %zu.\n", + msg->len, sizeof(*d)); + return; + } + + if (msg->len != sizeof(*d) + d->len) { + dprintk(KERN_ERR "Wrong message to crypto connector: msg->len=%u != %zu.\n", + msg->len, sizeof(*d) + d->len); + return; + } + + dev = crypto_device_get_name(d->name); + if (!dev) { + dprintk("Crypto device %s was not found.\n", d->name); + return; + } + + switch (d->cmd) { + case CRYPTO_GET_STAT: + reply = kmalloc(sizeof(*msg) + sizeof(*cmd) + sizeof(struct crypto_device_stat), GFP_ATOMIC); + if (reply) { + struct crypto_device_stat *ptr; + + memcpy(reply, msg, sizeof(*reply)); + reply->len = sizeof(*cmd) + sizeof(*ptr); + + /* + * See protocol description in connector.c + */ + reply->ack++; + + cmd = (struct crypto_conn_data *)(reply + 1); + memcpy(cmd, d, sizeof(*cmd)); + cmd->len = sizeof(*ptr); + + ptr = (struct crypto_device_stat *)(cmd + 1); + memcpy(ptr, &dev->stat, sizeof(*ptr)); + + cn_netlink_send(reply, 0, GFP_ATOMIC); + + kfree(reply); + } else + dprintk(KERN_ERR "Failed to allocate %zu bytes in reply to comamnd 0x%x.\n", + sizeof(*msg) + sizeof(*cmd), d->cmd); + break; + case CRYPTO_REQUEST: +#if 1 + { + struct crypto_user_direct *usr; + + usr = (struct crypto_user_direct *)(d->data); + + crypto_user_direct_add_request(msg->seq, msg->ack, usr); + } +#endif + break; + default: + dprintk(KERN_ERR "Wrong operation 0x%04x for crypto connector.\n", + d->cmd); + return; + } + + crypto_device_put(dev); +} + +int crypto_conn_init(void) +{ + int err; + + err = cn_add_callback(&crypto_conn_id, crypto_conn_name, crypto_conn_callback); + if (err) + return err; + + dprintk("Crypto connector callback is registered.\n"); + + return 0; +} + +void crypto_conn_fini(void) +{ + cn_del_callback(&crypto_conn_id); + dprintk("Crypto connector callback is unregistered.\n"); +} diff --git a/acrypto/crypto_conn.h b/acrypto/crypto_conn.h new file mode 100644 index 0000000..40bc3ac --- /dev/null +++ b/acrypto/crypto_conn.h @@ -0,0 +1,44 @@ +/* + * crypto_conn.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_CONN_H +#define __CRYPTO_CONN_H + +#include + +#define CRYPTO_REQUEST 1 +#define CRYPTO_GET_STAT 2 + +struct crypto_conn_data +{ + char name[SCACHE_NAMELEN]; + __u16 cmd; + __u16 len; + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +int crypto_conn_init(void); +void crypto_conn_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_CONN_H */ diff --git a/acrypto/crypto_dev.c b/acrypto/crypto_dev.c new file mode 100644 index 0000000..7b71eac --- /dev/null +++ b/acrypto/crypto_dev.c @@ -0,0 +1,440 @@ +/* + * crypto_dev.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +LIST_HEAD(cdev_list); +spinlock_t cdev_lock = SPIN_LOCK_UNLOCKED; +static u32 cdev_ids; + +static int crypto_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int crypto_probe(struct device *dev) +{ + return -ENODEV; +} + +static int crypto_remove(struct device *dev) +{ + return 0; +} + +static void crypto_release(struct device *dev) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, device); + + complete(&d->dev_released); +} + +static void crypto_class_release(struct class *class) +{ +} + +static void crypto_class_release_device(struct class_device *class_dev) +{ +} + +struct class crypto_class = { + .name = "acrypto", + .class_release = crypto_class_release, + .release = crypto_class_release_device +}; + +struct bus_type crypto_bus_type = { + .name = "acrypto", + .match = crypto_match +}; + +struct device_driver crypto_driver = { + .name = "crypto_driver", + .bus = &crypto_bus_type, + .probe = crypto_probe, + .remove = crypto_remove, +}; + +struct device crypto_dev = { + .parent = NULL, + .bus = &crypto_bus_type, + .bus_id = "Asynchronous crypto", + .driver = &crypto_driver, + .release = &crypto_release +}; + +static ssize_t sessions_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%d\n", atomic_read(&d->refcnt)); +} +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%s\n", d->name); +} +static ssize_t devices_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(d, &cdev_list, cdev_entry) { + off += sprintf(buf + off, "%s ", d->name); + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!off) + off = sprintf(buf, "No devices registered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} + +static ssize_t kmem_failed_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.kmem_failed); +} +static ssize_t sstarted_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sstarted); +} +static ssize_t sfinished_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.sfinished); +} +static ssize_t scompleted_show(struct class_device *dev, char *buf) +{ + struct crypto_device *d = container_of(dev, struct crypto_device, class_device); + + return sprintf(buf, "%llu\n", d->stat.scompleted); +} + +static CLASS_DEVICE_ATTR(sessions, 0444, sessions_show, NULL); +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(devices, 0444, devices_show, NULL); +static CLASS_DEVICE_ATTR(scompleted, 0444, scompleted_show, NULL); +static CLASS_DEVICE_ATTR(sstarted, 0444, sstarted_show, NULL); +static CLASS_DEVICE_ATTR(sfinished, 0444, sfinished_show, NULL); +static CLASS_DEVICE_ATTR(kmem_failed, 0444, kmem_failed_show, NULL); + +static void create_device_attributes(struct crypto_device *dev) +{ + class_device_create_file(&dev->class_device, &class_device_attr_sessions); + class_device_create_file(&dev->class_device, &class_device_attr_name); + class_device_create_file(&dev->class_device, &class_device_attr_scompleted); + class_device_create_file(&dev->class_device, &class_device_attr_sstarted); + class_device_create_file(&dev->class_device, &class_device_attr_sfinished); + class_device_create_file(&dev->class_device, &class_device_attr_kmem_failed); +} + +static void remove_device_attributes(struct crypto_device *dev) +{ + class_device_remove_file(&dev->class_device, &class_device_attr_sessions); + class_device_remove_file(&dev->class_device, &class_device_attr_name); + class_device_remove_file(&dev->class_device, &class_device_attr_scompleted); + class_device_remove_file(&dev->class_device, &class_device_attr_sstarted); + class_device_remove_file(&dev->class_device, &class_device_attr_sfinished); + class_device_remove_file(&dev->class_device, &class_device_attr_kmem_failed); +} + +int __match_initializer(struct crypto_capability *cap, struct crypto_session_initializer *ci, struct crypto_device *dev) +{ + unsigned int ref = atomic_read(&dev->refcnt) + 1; + int ret = 0; + + if (cap->operation == ci->operation && cap->type == ci->type && + cap->mode == (ci->mode & 0x1fff) && + cap->qlen >= ref) { + ret = 1; + dprintk("Match: %04x.%04x.%04x vs. %04x.%04x.%04x, max_len=%u, ref=%u.\n", + cap->operation, cap->type, cap->mode, + ci->operation, ci->type, ci->mode, + cap->qlen, ref); + } + + return ret; +} + +int match_initializer(struct crypto_device *dev, struct crypto_session_initializer *ci) +{ + int i; + + for (i = 0; i < dev->cap_number; ++i) { + struct crypto_capability *cap = &dev->cap[i]; + + if (__match_initializer(cap, ci, dev)) + return 1; + } + + return 0; +} + +void crypto_device_get(struct crypto_device *dev) +{ + atomic_inc(&dev->refcnt); +} + +struct crypto_device *crypto_device_get_name(char *name) +{ + struct crypto_device *dev; + int found = 0; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (!strcmp(dev->name, name)) { + found = 1; + crypto_device_get(dev); + break; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + if (!found) + return NULL; + + return dev; +} + +void crypto_device_put(struct crypto_device *dev) +{ + atomic_dec(&dev->refcnt); +} + +static int avg_cap_qlen(struct crypto_device *dev) +{ + int i, max = 0; + + if (!dev->cap_number) + return 0; + + for (i=0; icap_number; ++i) { + max += dev->cap[i].qlen; + } + + return (max / dev->cap_number); +} + +int __crypto_device_add(struct crypto_device *dev) +{ + int err = -ENOMEM, avg; + + memset(&dev->stat, 0, sizeof(dev->stat)); + spin_lock_init(&dev->stat_lock); + spin_lock_init(&dev->lock); + spin_lock_init(&dev->session_lock); + INIT_LIST_HEAD(&dev->session_list); + atomic_set(&dev->refcnt, 0); + atomic_set(&dev->sid, 0); + dev->flags = 0; + init_completion(&dev->dev_released); + memcpy(&dev->device, &crypto_dev, sizeof(struct device)); + dev->driver = &crypto_driver; + + dev->session_cache = kmem_cache_create(dev->name, sizeof(struct crypto_session), + 0, 0, NULL, NULL); + if (!dev->session_cache) { + dprintk(KERN_ERR "Failed to create session cache for device %s.\n", dev->name); + return -ENOMEM; + } + + avg = avg_cap_qlen(dev); + + if (avg) { + dev->session_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->session_cache); + if (!dev->session_pool) { + dprintk(KERN_ERR "Failed to create session memory pool with %d objects for device %s.\n", + avg, dev->name); + err = -ENOMEM; + goto err_out_cache_destroy; + } + } else + dev->session_pool = NULL; + + snprintf(dev->route_cache_name, sizeof(dev->route_cache_name), "%s-route", dev->name); + dev->route_cache = kmem_cache_create(dev->route_cache_name, sizeof(struct crypto_route), 0, 0, NULL, NULL); + if (!dev->route_cache) { + dprintk(KERN_ERR "Failed to create route cache for device %s.\n", dev->name); + goto err_out_mempool_destroy; + } + + avg = (avg)?avg*2:1024; + dev->route_pool = mempool_create(avg, mempool_alloc_slab, mempool_free_slab, dev->route_cache); + if (!dev->route_pool) { + dprintk(KERN_ERR "Failed to create route memory pool with %d objects for device %s.\n", + avg, dev->name); + err = -ENOMEM; + goto err_out_route_cache_destroy; + } + + snprintf(dev->device.bus_id, sizeof(dev->device.bus_id), "%s", dev->name); + err = device_register(&dev->device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto device %s: err=%d.\n", + dev->name, err); + goto err_out_route_mempool_destroy; + } + + snprintf(dev->class_device.class_id, sizeof(dev->class_device.class_id), "%s", dev->name); + dev->class_device.dev = &dev->device; + dev->class_device.class = &crypto_class; + + err = class_device_register(&dev->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto class device %s: err=%d.\n", + dev->name, err); + goto err_out_device_unregister; + } + + create_device_attributes(dev); + + return 0; + +err_out_device_unregister: + device_unregister(&dev->device); +err_out_route_mempool_destroy: + if (dev->route_pool) + mempool_destroy(dev->route_pool); +err_out_route_cache_destroy: + kmem_cache_destroy(dev->route_cache); +err_out_mempool_destroy: + if (dev->session_pool) + mempool_destroy(dev->session_pool); +err_out_cache_destroy: + kmem_cache_destroy(dev->session_cache); + + return err; +} + +void __crypto_device_remove(struct crypto_device *dev) +{ + remove_device_attributes(dev); + class_device_unregister(&dev->class_device); + device_unregister(&dev->device); + if (dev->route_pool) + mempool_destroy(dev->route_pool); + kmem_cache_destroy(dev->route_cache); + if (dev->session_pool) + mempool_destroy(dev->session_pool); + kmem_cache_destroy(dev->session_cache); +} + +int crypto_device_add(struct crypto_device *dev) +{ + int err; + unsigned long flags; + + err = __crypto_device_add(dev); + if (err) + return err; + + spin_lock_irqsave(&cdev_lock, flags); + list_add(&dev->cdev_entry, &cdev_list); + dev->id = ++cdev_ids; + spin_unlock_irqrestore(&cdev_lock, flags); + + printk("Crypto device %s was registered with ID=%x.\n", + dev->name, dev->id); + + return 0; +} + +void crypto_device_remove(struct crypto_device *dev) +{ + struct crypto_device *__dev, *n; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry_safe(__dev, n, &cdev_list, cdev_entry) { + if (compare_device(__dev, dev)) { + list_del_init(&__dev->cdev_entry); + spin_unlock_irqrestore(&cdev_lock, flags); + + + /* + * In test cases or when crypto device driver is not written correctly, + * it's ->data_ready() method will not be called anymore + * after device is removed from crypto device list. + * + * For such cases we either should provide ->flush() call + * or properly write ->data_ready() method. + */ + + while (atomic_read(&__dev->refcnt)) { + + dprintk("Waiting for %s to become free: refcnt=%d.\n", + __dev->name, atomic_read(&dev->refcnt)); + + /* + * Hack zone: you need to write good ->data_ready() + * and crypto device driver itself. + * + * Driver shoud not buzz if it has pending sessions + * in it's queue and it was removed from global device list. + * + * Although I can workaround it here, for example by + * flushing the whole queue and drop all pending sessions. + */ + + __dev->data_ready(__dev); + ssleep(1); + } + + __crypto_device_remove(dev); + + dprintk(KERN_ERR "Crypto device %s was unregistered.\n", + dev->name); + return; + } + } + spin_unlock_irqrestore(&cdev_lock, flags); + + dprintk(KERN_ERR "Crypto device %s was not registered.\n", dev->name); +} + +EXPORT_SYMBOL_GPL(crypto_device_add); +EXPORT_SYMBOL_GPL(crypto_device_remove); +EXPORT_SYMBOL_GPL(crypto_device_get); +EXPORT_SYMBOL_GPL(crypto_device_get_name); +EXPORT_SYMBOL_GPL(crypto_device_put); +EXPORT_SYMBOL_GPL(match_initializer); diff --git a/acrypto/crypto_lb.c b/acrypto/crypto_lb.c new file mode 100644 index 0000000..16d59f4 --- /dev/null +++ b/acrypto/crypto_lb.c @@ -0,0 +1,481 @@ +/* + * crypto_lb.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" +#include "crypto_stat.h" +#include "crypto_route.h" + +static LIST_HEAD(crypto_lb_list); +static spinlock_t crypto_lb_lock = SPIN_LOCK_UNLOCKED; +static int lb_num = 0; +static struct crypto_lb *current_lb, *default_lb; +static struct workqueue_struct *crypto_lb_queue; + +extern struct list_head cdev_list; +extern spinlock_t cdev_lock; + +extern struct crypto_lb simple_lb; + +static int lb_is_current(struct crypto_lb *l) +{ + return (l == current_lb); +} + +static int lb_is_default(struct crypto_lb *l) +{ + return (l == default_lb); +} + +#if 0 +static void lb_set_current(struct crypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + current_lb = l; + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} + +static void lb_set_default(struct crypto_lb *l) +{ + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + default_lb = l; + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} +#endif +static int crypto_lb_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int crypto_lb_probe(struct device *dev) +{ + return -ENODEV; +} + +static int crypto_lb_remove(struct device *dev) +{ + return 0; +} + +static void crypto_lb_release(struct device *dev) +{ + struct crypto_lb *d = container_of(dev, struct crypto_lb, device); + + complete(&d->dev_released); +} + +static void crypto_lb_class_release(struct class *class) +{ +} + +static void crypto_lb_class_release_device(struct class_device *class_dev) +{ +} + +struct class crypto_lb_class = { + .name = "crypto_lb", + .class_release = crypto_lb_class_release, + .release = crypto_lb_class_release_device +}; + +struct bus_type crypto_lb_bus_type = { + .name = "crypto_lb", + .match = crypto_lb_match +}; + +struct device_driver crypto_lb_driver = { + .name = "crypto_lb_driver", + .bus = &crypto_lb_bus_type, + .probe = crypto_lb_probe, + .remove = crypto_lb_remove, +}; + +struct device crypto_lb_dev = { + .parent = NULL, + .bus = &crypto_lb_bus_type, + .bus_id = "crypto load balancer", + .driver = &crypto_lb_driver, + .release = &crypto_lb_release +}; + +static ssize_t name_show(struct class_device *dev, char *buf) +{ + struct crypto_lb *lb = container_of(dev, struct crypto_lb, class_device); + + return sprintf(buf, "%s\n", lb->name); +} + +static ssize_t current_show(struct class_device *dev, char *buf) +{ + struct crypto_lb *lb; + int off = 0; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(lb, &crypto_lb_list, lb_entry) { + if (lb_is_current(lb)) + off += sprintf(buf + off, "["); + if (lb_is_default(lb)) + off += sprintf(buf + off, "("); + off += sprintf(buf + off, "%s", lb->name); + if (lb_is_default(lb)) + off += sprintf(buf + off, ")"); + if (lb_is_current(lb)) + off += sprintf(buf + off, "]"); + } + + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + if (!off) + off = sprintf(buf, "No load balancers regitered yet."); + + off += sprintf(buf + off, "\n"); + + return off; +} +static ssize_t current_store(struct class_device *dev, const char *buf, size_t count) +{ + struct crypto_lb *lb; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(lb, &crypto_lb_list, lb_entry) { + if (count == strlen(lb->name) && !strcmp(buf, lb->name)) { + current_lb = lb; + default_lb = lb; + + dprintk("Load balancer %s is set as current and default.\n", + lb->name); + + break; + } + } + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + return count; +} + +static CLASS_DEVICE_ATTR(name, 0444, name_show, NULL); +CLASS_DEVICE_ATTR(lbs, 0644, current_show, current_store); + +static void create_device_attributes(struct crypto_lb *lb) +{ + class_device_create_file(&lb->class_device, &class_device_attr_name); +} + +static void remove_device_attributes(struct crypto_lb *lb) +{ + class_device_remove_file(&lb->class_device, &class_device_attr_name); +} + +static int compare_lb(struct crypto_lb *l1, struct crypto_lb *l2) +{ + if (!strncmp(l1->name, l2->name, sizeof(l1->name))) + return 1; + + return 0; +} + +struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_device *dev; + unsigned long flags; + + if (!current_lb) + return NULL; + + if (sci_bound(ci)) { + int found = 0; + + spin_lock_irqsave(&cdev_lock, flags); + + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (dev->id == ci->bdev) { + found = 1; + break; + } + } + + spin_unlock_irqrestore(&cdev_lock, flags); + + return (found) ? dev : NULL; + } + + spin_lock_irqsave(¤t_lb->lock, flags); + dev = current_lb->find_device(current_lb, ci, data); + spin_unlock_irqrestore(¤t_lb->lock, flags); + + return dev; +} + +static int __crypto_lb_register(struct crypto_lb *lb) +{ + int err; + + spin_lock_init(&lb->lock); + + init_completion(&lb->dev_released); + memcpy(&lb->device, &crypto_lb_dev, sizeof(struct device)); + lb->driver = &crypto_lb_driver; + + snprintf(lb->device.bus_id, sizeof(lb->device.bus_id), "%s", lb->name); + err = device_register(&lb->device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer device %s: err=%d.\n", + lb->name, err); + return err; + } + + snprintf(lb->class_device.class_id, sizeof(lb->class_device.class_id), "%s", lb->name); + lb->class_device.dev = &lb->device; + lb->class_device.class = &crypto_lb_class; + + err = class_device_register(&lb->class_device); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer class device %s: err=%d.\n", + lb->name, err); + device_unregister(&lb->device); + return err; + } + + create_device_attributes(lb); + + return 0; + +} + +void __crypto_lb_unregister(struct crypto_lb *lb) +{ + remove_device_attributes(lb); + class_device_unregister(&lb->class_device); + device_unregister(&lb->device); +} + +int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default) +{ + struct crypto_lb *__lb; + int err; + unsigned long flags; + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry(__lb, &crypto_lb_list, lb_entry) { + if (unlikely(compare_lb(__lb, lb))) { + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + dprintk(KERN_ERR "Crypto load balancer %s is already registered.\n", + lb->name); + return -EINVAL; + } + } + + list_add(&lb->lb_entry, &crypto_lb_list); + + if (!default_lb || set_default) + default_lb = lb; + + if (!current_lb || set_current) + current_lb = lb; + + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + err = __crypto_lb_register(lb); + if (err) { + spin_lock_irqsave(&crypto_lb_lock, flags); + list_del_init(&lb->lb_entry); + current_lb = &simple_lb; + default_lb = &simple_lb; + spin_unlock_irqrestore(&crypto_lb_lock, flags); + + return err; + } + + dprintk("Crypto load balancer %s was registered and set to be [%s.%s].\n", + lb->name, (lb_is_current(lb)) ? "current" : "not current", + (lb_is_default(lb)) ? "default" : "not default"); + + lb_num++; + + return 0; +} + +void crypto_lb_unregister(struct crypto_lb *lb) +{ + struct crypto_lb *__lb, *n; + unsigned long flags; + + __crypto_lb_unregister(lb); + + spin_lock_irqsave(&crypto_lb_lock, flags); + + list_for_each_entry_safe(__lb, n, &crypto_lb_list, lb_entry) { + if (compare_lb(__lb, lb)) { + lb_num--; + list_del_init(&__lb->lb_entry); + + dprintk("Crypto load balancer %s was unregistered.\n", lb->name); + break; + } + } + + spin_unlock_irqrestore(&crypto_lb_lock, flags); +} + +static void crypto_lb_queue_wrapper(void *data) +{ + struct crypto_session *s = (struct crypto_session *)data; + + dprintk("%s: Calling callback for session %08x [%08x] flags=%lx, " + "op=%04u, type=%04x, mode=%04x, priority=%04x\n", __func__, + s->ci.id, s->ci.dev_id, s->ci.flags, s->ci.operation, + s->ci.type, s->ci.mode, s->ci.priority); + + s->ci.callback(&s->ci, &s->data); + + if (likely(session_completed(s))) { + crypto_session_destroy(s); + return; + } else { + /* + * Not implemented yet. + */ + /* + * Special case: crypto consumer marks session as "not finished" + * in it's callback - it means that crypto consumer wants + * this session to be processed further, + * for example crypto consumer can add new route and then + * mark session as "not finished". + */ + BUG(); + } +} + +static void crypto_lb_process_next_route(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_copy_current_ci(s); + if (dev) + __crypto_session_add(dev, s); +} + +int complete_session(struct crypto_session *s) +{ + crypto_stat_complete_inc(s); + crypto_stat_ptime_inc(s); + set_bit(SESSION_COMPLETED, &s->ci.flags); + crypto_session_dequeue_route(s); + + if (unlikely(crypto_route_queue_len(s))) { + crypto_lb_process_next_route(s); + } else { + if (test_bit(SESSION_DIRECT, &s->ci.flags)) + crypto_lb_queue_wrapper(s); + else { + INIT_WORK(&s->work, &crypto_lb_queue_wrapper, s); + queue_work(crypto_lb_queue, &s->work); + } + } + + return 0; +} + +int crypto_lb_init(void) +{ + int err; + + err = bus_register(&crypto_lb_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer bus: err=%d.\n", err); + goto err_out_exit; + } + + err = driver_register(&crypto_lb_driver); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer driver: err=%d.\n", err); + goto err_out_bus_unregister; + } + + crypto_lb_class.class_dev_attrs = &class_device_attr_lbs; + + err = class_register(&crypto_lb_class); + if (err) { + dprintk(KERN_ERR "Failed to register crypto load balancer class: err=%d.\n", err); + goto err_out_driver_unregister; + } + + crypto_lb_queue = create_singlethread_workqueue("clbq"); + if (!crypto_lb_queue) { + dprintk(KERN_ERR "Failed to create crypto load balaner work queue.\n"); + goto err_out_class_unregister; + } + + err = crypto_lb_register(&simple_lb, 1, 1); + if (err) + goto err_out_destroy_workqueue; + + return 0; + +err_out_destroy_workqueue: + destroy_workqueue(crypto_lb_queue); +err_out_class_unregister: + class_unregister(&crypto_lb_class); +err_out_driver_unregister: + driver_unregister(&crypto_lb_driver); +err_out_bus_unregister: + bus_unregister(&crypto_lb_bus_type); +err_out_exit: + return err; +} + +void crypto_lb_fini(void) +{ + __crypto_lb_unregister(&simple_lb); + + flush_workqueue(crypto_lb_queue); + destroy_workqueue(crypto_lb_queue); + class_unregister(&crypto_lb_class); + driver_unregister(&crypto_lb_driver); + bus_unregister(&crypto_lb_bus_type); +} + +EXPORT_SYMBOL_GPL(crypto_lb_register); +EXPORT_SYMBOL_GPL(crypto_lb_unregister); +EXPORT_SYMBOL_GPL(crypto_lb_find_device); diff --git a/acrypto/crypto_lb.h b/acrypto/crypto_lb.h new file mode 100644 index 0000000..1539d45 --- /dev/null +++ b/acrypto/crypto_lb.h @@ -0,0 +1,56 @@ +/* + * crypto_lb.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_LB_H +#define __CRYPTO_LB_H + +#include + +#define CRYPTO_LB_NAMELEN 32 + +struct crypto_lb +{ + struct list_head lb_entry; + + char name[CRYPTO_LB_NAMELEN]; + + struct crypto_device * (*find_device) (struct crypto_lb *, + struct crypto_session_initializer *, + struct crypto_data *); + + spinlock_t lock; + + struct device_driver *driver; + struct device device; + struct class_device class_device; + struct completion dev_released; + +}; + +int crypto_lb_register(struct crypto_lb *lb, int set_current, int set_default); +void crypto_lb_unregister(struct crypto_lb *); + +struct crypto_device *crypto_lb_find_device(struct crypto_session_initializer *, struct crypto_data *); + +int crypto_lb_init(void); +void crypto_lb_fini(void); + +#endif /* __CRYPTO_LB_H */ diff --git a/acrypto/crypto_main.c b/acrypto/crypto_main.c new file mode 100644 index 0000000..dc4ec35 --- /dev/null +++ b/acrypto/crypto_main.c @@ -0,0 +1,390 @@ +/* + * crypto_main.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_lb.h" +#include "crypto_conn.h" +#include "crypto_route.h" +#include "crypto_user_ioctl.h" +#include "crypto_stat.h" + +extern struct bus_type crypto_bus_type; +extern struct device_driver crypto_driver; +extern struct class crypto_class; +extern struct device crypto_dev; + +static atomic_t global_session_id = ATOMIC_INIT(0); + +int __crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s) +{ + list_add_tail(&s->dev_queue_entry, &dev->session_list); + return 0; +} + +int crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&dev->session_lock, flags); + err = __crypto_session_enqueue(dev, s); + spin_unlock_irqrestore(&dev->session_lock, flags); + + return err; +} + +void crypto_session_destroy(struct crypto_session *s) +{ + if (s->data.priv_size && s->data.priv) + kfree(s->data.priv); + + if (session_from_cache(s)) + kmem_cache_free(s->pool_dev->session_cache, s); + else + mempool_free(s, s->pool_dev->session_pool); +} + +struct crypto_session *crypto_session_create(struct crypto_session_initializer *ci, struct crypto_data *d) +{ + struct crypto_device *ldev; + struct crypto_session *s; + int err; + + if (d->priv_size > CRYPTO_MAX_PRIV_SIZE) { + dprintk("priv_size %u is too big, maximum allowed %u.\n", + d->priv_size, CRYPTO_MAX_PRIV_SIZE); + return NULL; + } + + /* + * We grab a reference here if we can find device. + * This reference will be dropped when route will be removed. + */ + ldev = crypto_lb_find_device(ci, d); + if (!ldev) { + dprintk("Cannot find suitable device for [%02x.%02x.%02x.%02x].\n", + ci->operation, ci->mode, ci->type, ci->priority); + return NULL; + } + + s = mempool_alloc(ldev->session_pool, GFP_ATOMIC); + if (!s) { + ldev->stat.pool_failed++; + + s = kmem_cache_alloc(ldev->session_cache, GFP_ATOMIC); + if (!s) { + ldev->stat.kmem_failed++; + goto err_out_device_put; + } + + mark_session_from_cache(s); + } + + s->pool_dev = ldev; + + crypto_route_head_init(&s->route_list); + INIT_LIST_HEAD(&s->dev_queue_entry); + + spin_lock_init(&s->lock); + + memcpy(&s->ci, ci, sizeof(s->ci)); + memcpy(&s->data, d, sizeof(s->data)); + + s->data.priv = NULL; + if (d->priv_size) { + s->data.priv = kmalloc(d->priv_size, GFP_ATOMIC); + if (!s->data.priv) { + ldev->stat.kmem_failed++; + goto err_out_session_free; + } + + if (d->priv) + memcpy(s->data.priv, d->priv, d->priv_size); + } + else + s->data.priv = d->priv; + + s->ci.id = atomic_inc_return(&global_session_id); + s->ci.dev_id = atomic_inc_return(&ldev->sid); + s->ci.flags = 0; + + err = crypto_route_add_direct(ldev, s, ci); + if (err) { + ldev->stat.route_failed++; + dprintk("Can not add route to device %s.\n", ldev->name); + goto err_out_session_free; + } + + return s; + +err_out_session_free: + crypto_session_destroy(s); +err_out_device_put: + crypto_device_put(ldev); + + return NULL; +} + +int __crypto_session_add(struct crypto_device *ldev, struct crypto_session *s) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&s->lock, flags); + err = crypto_session_enqueue(ldev, s); + spin_unlock_irqrestore(&s->lock, flags); + + if (!err && ldev->data_ready) + ldev->data_ready(ldev); + + return err; +} + +int crypto_session_add(struct crypto_session *s) +{ + struct crypto_device *ldev; + + ldev = crypto_route_get_current_device(s); + if (!ldev) + return -ENODEV; + + return __crypto_session_add(ldev, s); +} + +struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *ci, struct crypto_data *d) +{ + struct crypto_session *s; + int err; + + s = crypto_session_create(ci, d); + if (!s) + return NULL; + + err = crypto_session_add(s); + if (err) { + crypto_session_destroy(s); + return NULL; + } + + return s; +} + +int crypto_session_dequeue_route(struct crypto_session *s) +{ + struct crypto_route *rt; + + rt = crypto_route_dequeue(s); + if (rt) { + crypto_route_free(rt); + return 0; + } + + return -ENODEV; +} + +struct crypto_session *crypto_session_dequeue(struct crypto_device *dev) +{ + unsigned long flags; + struct crypto_session *s = NULL; + + spin_lock_irqsave(&dev->session_lock, flags); + if (list_empty(&dev->session_list)) + goto unlock; + + s = list_entry(dev->session_list.next, struct crypto_session, dev_queue_entry); + list_del(&s->dev_queue_entry); +unlock: + spin_unlock_irqrestore(&dev->session_lock, flags); + + return s; +} + +void crypto_process(struct crypto_session *s, + int (*proc)(struct crypto_session *s, void *src, int slen, void *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + int (*proc_scatter)(struct crypto_session *s, struct scatterlist *src, int slen, + struct scatterlist *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + void (*finish)(struct crypto_session *s, void *data), + void *data) +{ + int i, keylen, ivlen, err = -EINVAL; + void *key, *iv; + + if (session_completed(s)) + return; + + key = iv = NULL; + ivlen = keylen = 0; + + err = -ENOMEM; + /* + * Simple case - key is small(it's size is less than PAGE_SIZE). + * Assymetric crypto will require proper key sg handling. + */ + if (s->data.sg_key[0].length) { + key = kmap(s->data.sg_key[0].page); + if (!key) + goto err_out_exit; + key += s->data.sg_key[0].offset; + keylen = s->data.sg_key[0].length; + } + + if (s->data.sg_iv[0].length) { + iv = kmap(s->data.sg_iv[0].page); + if (!iv) + goto err_out_unmap_key; + iv += s->data.sg_iv[0].offset; + ivlen = s->data.sg_iv[0].length; + } + + for (i=0; idata.sg_src_num; ++i) { + int dlen, slen; + + slen = s->data.sg_src[i].length; + dlen = s->data.sg_dst[i].length; + + if (proc) { + u8 *dst, *src; + + dst = kmap_atomic(s->data.sg_dst[i].page, KM_USER0) + s->data.sg_dst[i].offset; + src = kmap_atomic(s->data.sg_src[i].page, KM_USER1) + s->data.sg_src[i].offset; + + err = proc(s, src, slen, dst, dlen, key, keylen, iv, ivlen, data); + + kunmap_atomic(src, KM_USER1); + kunmap_atomic(dst, KM_USER0); + } else if (proc_scatter) + err = proc_scatter(s, &s->data.sg_src[i], slen, &s->data.sg_dst[i], dlen, key, keylen, iv, ivlen, data); + else + err = -EINVAL; + + s->data.sg_dst[i].length = s->data.sg_src[i].length; + s->data.sg_dst[i].offset = s->data.sg_src[i].offset; + + if (err < 0) { + break_session(s); + dprintk("operation=%02x, size=%u, err=%d.\n", s->ci.operation, s->data.sg_src[i].length, err); + break; + } + } + + if (finish) + finish(s, data); + + if (iv) + kunmap(s->data.sg_iv[0].page); +err_out_unmap_key: + if (key) + kunmap(s->data.sg_key[0].page); +err_out_exit: + + complete_session(s); +} + +int __devinit cmain_init(void) +{ + int err; + + err = bus_register(&crypto_bus_type); + if (err) { + dprintk(KERN_ERR "Failed to register crypto bus: err=%d.\n", + err); + return err; + } + + err = driver_register(&crypto_driver); + if (err) { + dprintk(KERN_ERR "Failed to register crypto driver: err=%d.\n", + err); + goto err_out_bus_unregister; + } + + err = class_register(&crypto_class); + if (err) { + dprintk(KERN_ERR "Failed to register crypto class: err=%d.\n", + err); + goto err_out_driver_unregister; + } + + err = crypto_lb_init(); + if (err) + goto err_out_class_unregister; + + err = crypto_conn_init(); + if (err) + goto err_out_crypto_lb_fini; + + err = crypto_user_ioctl_init(); + if (err) + goto err_out_crypto_conn_fini; + + return 0; + +err_out_crypto_conn_fini: + crypto_conn_fini(); +err_out_crypto_lb_fini: + crypto_lb_fini(); +err_out_class_unregister: + class_unregister(&crypto_class); +err_out_driver_unregister: + driver_unregister(&crypto_driver); +err_out_bus_unregister: + bus_unregister(&crypto_bus_type); + + return err; +} + +void __devexit cmain_fini(void) +{ + crypto_user_ioctl_fini(); + + crypto_conn_fini(); + crypto_lb_fini(); + + class_unregister(&crypto_class); + driver_unregister(&crypto_driver); + bus_unregister(&crypto_bus_type); +} + +module_init(cmain_init); +module_exit(cmain_fini); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Evgeniy Polyakov "); +MODULE_DESCRIPTION("Asynchronous crypto layer."); + +EXPORT_SYMBOL(crypto_session_alloc); +EXPORT_SYMBOL_GPL(crypto_session_create); +EXPORT_SYMBOL_GPL(crypto_session_add); +EXPORT_SYMBOL_GPL(__crypto_session_add); +EXPORT_SYMBOL_GPL(crypto_process); +EXPORT_SYMBOL_GPL(crypto_session_dequeue_route); +EXPORT_SYMBOL_GPL(crypto_session_dequeue); diff --git a/acrypto/crypto_route.h b/acrypto/crypto_route.h new file mode 100644 index 0000000..aee7e23 --- /dev/null +++ b/acrypto/crypto_route.h @@ -0,0 +1,273 @@ +/* + * crypto_route.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_ROUTE_H +#define __CRYPTO_ROUTE_H + +#include +#include +#include + +#include + +static inline struct crypto_route *crypto_route_alloc_direct(struct crypto_device *dev, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = mempool_alloc(dev->route_pool, GFP_ATOMIC); + if (!rt) + return NULL; + + memset(rt, 0, sizeof(*rt)); + memcpy(&rt->ci, ci, sizeof(*ci)); + + rt->dev = dev; + + return rt; +} + +static inline struct crypto_route *crypto_route_alloc(struct crypto_device *dev, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + if (!match_initializer(dev, ci)) + return NULL; + + rt = crypto_route_alloc_direct(dev, ci); + + return rt; +} + +/* + * Device's reference counter must be incremented before routing was + * allocated, so routing subsystem itself does not care about it. + */ +static inline void crypto_route_free(struct crypto_route *rt) +{ + struct crypto_device *dev = rt->dev; + + mempool_free(rt, dev->route_pool); + crypto_device_put(dev); +} + +static inline void __crypto_route_del(struct crypto_route *rt, struct crypto_route_head *list) +{ + struct crypto_route *next, *prev; + + list->qlen--; + next = rt->next; + prev = rt->prev; + rt->next = rt->prev = NULL; + rt->list = NULL; + next->prev = prev; + prev->next = next; +} + +static inline void crypto_route_del(struct crypto_route *rt) +{ + struct crypto_route_head *list = rt->list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + if (list == rt->list) + __crypto_route_del(rt, rt->list); + spin_unlock_irqrestore(&list->lock, flags); + + crypto_route_free(rt); + } +} + +static inline struct crypto_route *__crypto_route_dequeue(struct crypto_route_head *list) +{ + struct crypto_route *next, *prev, *result; + + prev = (struct crypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) { + result = next; + next = next->next; + list->qlen--; + next->prev = prev; + prev->next = next; + result->next = result->prev = NULL; + result->list = NULL; + } + return result; +} + +static inline struct crypto_route *crypto_route_dequeue(struct crypto_session *s) +{ + struct crypto_route *rt; + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + rt = __crypto_route_dequeue(&s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); + + return rt; +} + +static inline void __crypto_route_queue(struct crypto_route *rt, struct crypto_route_head *list) +{ + struct crypto_route *prev, *next; + + rt->list = list; + list->qlen++; + next = (struct crypto_route *)list; + prev = next->prev; + rt->next = next; + rt->prev = prev; + next->prev = prev->next = rt; +} + +static inline void crypto_route_queue(struct crypto_route *rt, struct crypto_session *s) +{ + unsigned long flags; + + spin_lock_irqsave(&s->route_list.lock, flags); + __crypto_route_queue(rt, &s->route_list); + spin_unlock_irqrestore(&s->route_list.lock, flags); +} + +/* + * Caller must hold a reference for crypto_device being added, + * and it is not permitted to drop the reference until routing is removed. + * See crypto_session_create() for example of right device's reference counter usage. + */ + +static inline int crypto_route_add(struct crypto_device *dev, struct crypto_session *s, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = crypto_route_alloc(dev, ci); + if (!rt) + return -ENOMEM; + + crypto_route_queue(rt, s); + + return 0; +} + +static inline int crypto_route_add_direct(struct crypto_device *dev, struct crypto_session *s, + struct crypto_session_initializer *ci) +{ + struct crypto_route *rt; + + rt = crypto_route_alloc_direct(dev, ci); + if (!rt) + return -ENOMEM; + + crypto_route_queue(rt, s); + + return 0; +} + +static inline int crypto_route_queue_len(struct crypto_session *s) +{ + return s->route_list.qlen; +} + +static inline void crypto_route_head_init(struct crypto_route_head *list) +{ + spin_lock_init(&list->lock); + list->prev = list->next = (struct crypto_route *)list; + list->qlen = 0; +} + +static inline struct crypto_route *__crypto_route_current(struct crypto_route_head *list) +{ + struct crypto_route *next, *prev, *result; + + prev = (struct crypto_route *)list; + next = prev->next; + result = NULL; + if (next != prev) + result = next; + + return result; +} + +static inline struct crypto_route *crypto_route_current(struct crypto_session *s) +{ + struct crypto_route_head *list; + struct crypto_route *rt = NULL; + + list = &s->route_list; + + if (list) { + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __crypto_route_current(list); + spin_unlock_irqrestore(&list->lock, flags); + } + + return rt; +} + +static inline struct crypto_device *crypto_route_copy_current_ci(struct crypto_session *s) +{ + struct crypto_route *rt; + struct crypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + + rt = __crypto_route_current(list); + if (rt) { + memcpy(&s->ci, &rt->ci, sizeof(s->ci)); + return rt->dev; + } + + spin_unlock_irqrestore(&list->lock, flags); + + return NULL; +} + +/* + * It is not allowed to use returned device pointer + * after current route is removed. + * If you want to use current device and remove current route, + * you MUST grab the reference before removing the route. + */ +static inline struct crypto_device *crypto_route_get_current_device(struct crypto_session *s) +{ + struct crypto_route *rt; + struct crypto_device *dev = NULL; + struct crypto_route_head *list = &s->route_list; + unsigned long flags; + + spin_lock_irqsave(&list->lock, flags); + rt = __crypto_route_current(list); + if (rt) + dev = rt->dev; + spin_unlock_irqrestore(&list->lock, flags); + + return dev; +} + +#endif /* __CRYPTO_ROUTE_H */ diff --git a/acrypto/crypto_stat.c b/acrypto/crypto_stat.c new file mode 100644 index 0000000..a6f5fe0 --- /dev/null +++ b/acrypto/crypto_stat.c @@ -0,0 +1,69 @@ +/* + * crypto_stat.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_route.h" + +void crypto_stat_complete_inc(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_get_current_device(s); + if (dev) { + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + dev->stat.scompleted++; + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +void crypto_stat_ptime_inc(struct crypto_session *s) +{ + struct crypto_device *dev; + + dev = crypto_route_get_current_device(s); + if (dev) { + int i; + unsigned long flags; + + spin_lock_irqsave(&dev->stat_lock, flags); + for (i = 0; i < dev->cap_number; ++i) { + if (__match_initializer(&dev->cap[i], &s->ci, dev)) { + dev->cap[i].ptime += s->ci.ptime; + dev->cap[i].scomp++; + break; + } + } + spin_unlock_irqrestore(&dev->stat_lock, flags); + } +} + +EXPORT_SYMBOL(crypto_stat_complete_inc); diff --git a/acrypto/crypto_stat.h b/acrypto/crypto_stat.h new file mode 100644 index 0000000..4894b44 --- /dev/null +++ b/acrypto/crypto_stat.h @@ -0,0 +1,30 @@ +/* + * crypto_stat.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_STAT_H +#define __CRYPTO_STAT_H + +#include + +void crypto_stat_complete_inc(struct crypto_session *s); +void crypto_stat_ptime_inc(struct crypto_session *s); + +#endif /* __CRYPTO_STAT_H */ diff --git a/acrypto/crypto_user.c b/acrypto/crypto_user.c new file mode 100644 index 0000000..ab9b80e --- /dev/null +++ b/acrypto/crypto_user.c @@ -0,0 +1,196 @@ +/* + * crypto_user.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto_user.h" + +static struct scatterlist *crypto_user_alloc_sg(int sg_num) +{ + struct scatterlist *sg; + + sg = kmalloc(sg_num * sizeof(*sg), GFP_ATOMIC); + if (!sg) { + dprintk("Failed to allocate %d scatterlist structures.\n", sg_num); + return NULL; + } + + memset(sg, 0, sizeof(*sg) * sg_num); + + return sg; +} + +static void crypto_user_free_sg(struct scatterlist *sg) +{ + kfree(sg); +} + +int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size) +{ + int sg_num; + + if ( (src_size > MAX_DATA_SIZE * PAGE_SIZE) || + (dst_size > MAX_DATA_SIZE * PAGE_SIZE) || + (key_size > MAX_DATA_SIZE * PAGE_SIZE) || + (iv_size > MAX_DATA_SIZE * PAGE_SIZE)) { + dprintk("Sizes are too big: src=%u, dst=%u, key=%u, iv=%u, max=%u.\n", + src_size, dst_size, key_size, iv_size, MAX_DATA_SIZE); + return -EINVAL; + + } + + sg_num = ALIGN_DATA_SIZE(src_size) / PAGE_SIZE; + data->sg_src = crypto_user_alloc_sg(sg_num); + if (!data->sg_src) + goto err_out_exit; + data->sg_src_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(dst_size) / PAGE_SIZE; + data->sg_dst = crypto_user_alloc_sg(sg_num); + if (!data->sg_dst) + goto err_out_free_src; + data->sg_dst_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(key_size) / PAGE_SIZE; + data->sg_key = crypto_user_alloc_sg(sg_num); + if (!data->sg_key) + goto err_out_free_dst; + data->sg_key_num = sg_num; + + sg_num = ALIGN_DATA_SIZE(iv_size) / PAGE_SIZE; + data->sg_iv = crypto_user_alloc_sg(sg_num); + if (!data->sg_iv) + goto err_out_free_key; + data->sg_iv_num = sg_num; + + return 0; + +err_out_free_key: + crypto_user_free_sg(data->sg_key); +err_out_free_dst: + crypto_user_free_sg(data->sg_dst); +err_out_free_src: + crypto_user_free_sg(data->sg_src); +err_out_exit: + + return -ENOMEM; +} + +void crypto_user_free_crypto_data(struct crypto_data *data) +{ + crypto_user_free_sg(data->sg_src); + crypto_user_free_sg(data->sg_dst); + crypto_user_free_sg(data->sg_key); + crypto_user_free_sg(data->sg_iv); +} + +void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg) +{ + int i, sg_num; + + sg_num = ALIGN_DATA_SIZE(size) / PAGE_SIZE; + + dprintk("Filling %d sgs, total size %u: ", sg_num, size); + + for (i=0; i size) + sg[i].length = size; + } else { + sg[i].offset = 0; + sg[i].length = (i != sg_num-1)?PAGE_SIZE:size; + } + dprintka("%x.%x.%p.%lx ", sg[i].offset, sg[i].length, ptr, ALIGN_DATA_SIZE((unsigned long)ptr)); + + size -= sg[i].length; + ptr += sg[i].length; + } + dprintka("\n"); +} + +struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data) +{ + struct scatterlist *sg = NULL; + int inval = 0; + + switch (ud->data_type) { + case CRYPTO_USER_DATA_SRC: + sg = data->sg_src; + inval = (data->sg_src_num * PAGE_SIZE < ud->data_size); + dprintk("Found SRC data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_DST: + sg = data->sg_dst; + inval = (data->sg_dst_num * PAGE_SIZE < ud->data_size); + dprintk("Found DST data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_KEY: + sg = data->sg_key; + inval = (data->sg_key_num * PAGE_SIZE < ud->data_size); + dprintk("Found KEY data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + case CRYPTO_USER_DATA_IV: + sg = data->sg_iv; + inval = (data->sg_iv_num * PAGE_SIZE < ud->data_size); + dprintk("Found IV data type, inval=%d, size=%u.\n", inval, ud->data_size); + break; + default: + dprintk("Unknown data type 0x%x, size=%u.\n", ud->data_type, ud->data_size); + break; + } + + return (inval)?NULL:sg; +} + +int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr) +{ + struct scatterlist *sg; + + sg = crypto_user_get_sg(ud, data); + if (!sg) + return -EINVAL; + + crypto_user_fill_sg(ptr, ud->data_size, sg); + + return 0; +} + +EXPORT_SYMBOL_GPL(crypto_user_alloc_crypto_data); +EXPORT_SYMBOL_GPL(crypto_user_free_crypto_data); +EXPORT_SYMBOL_GPL(crypto_user_fill_sg); +EXPORT_SYMBOL_GPL(crypto_user_fill_sg_data); +EXPORT_SYMBOL_GPL(crypto_user_get_sg); diff --git a/acrypto/crypto_user.h b/acrypto/crypto_user.h new file mode 100644 index 0000000..c08cde7 --- /dev/null +++ b/acrypto/crypto_user.h @@ -0,0 +1,52 @@ +/* + * crypto_user.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_H +#define __CRYPTO_USER_H + +#define MAX_DATA_SIZE 3 +#define ALIGN_DATA_SIZE(size) ((size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) + +enum crypto_user_data_types +{ + CRYPTO_USER_DATA_SRC = 0, + CRYPTO_USER_DATA_DST, + CRYPTO_USER_DATA_KEY, + CRYPTO_USER_DATA_IV, +}; + +struct crypto_user_data +{ + __u16 data_size; + __u16 data_type; +}; + + +#ifdef __KERNEL__ + +int crypto_user_alloc_crypto_data(struct crypto_data *data, int src_size, int dst_size, int key_size, int iv_size); +void crypto_user_free_crypto_data(struct crypto_data *data); +void crypto_user_fill_sg(void *ptr, u16 size, struct scatterlist *sg); +struct scatterlist *crypto_user_get_sg(struct crypto_user_data *ud, struct crypto_data *data); +int crypto_user_fill_sg_data(struct crypto_user_data *ud, struct crypto_data *data, void *ptr); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_USER_H */ diff --git a/acrypto/crypto_user_direct.c b/acrypto/crypto_user_direct.c new file mode 100644 index 0000000..626e9f7 --- /dev/null +++ b/acrypto/crypto_user_direct.c @@ -0,0 +1,389 @@ +/* + * crypto_user_direct.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "crypto_user.h" +#include "crypto_user_direct.h" + +extern struct cb_id crypto_conn_id; + +static LIST_HEAD(crypto_user_direct_list); +static spinlock_t crypto_user_direct_lock = SPIN_LOCK_UNLOCKED; +static struct completion thread_exited; +static int need_exit; +static DECLARE_WAIT_QUEUE_HEAD(crypto_user_direct_wait_queue); +static int crypto_user_direct_has_work; + +static int crypto_user_direct_alloc_pages(struct crypto_user_direct_kern *k) +{ + k->sp = kmalloc(sizeof(struct page *) * k->snum, GFP_KERNEL); + if (!k->sp) { + dprintk("Failed to allocate %d source pages.\n", k->snum); + return -ENOMEM; + } + + k->dp = kmalloc(sizeof(struct page *) * k->dnum, GFP_KERNEL); + if (!k->dp) { + dprintk("Failed to allocate %d destination pages.\n", k->dnum); + kfree(k->sp); + return -ENOMEM; + } + + return 0; +} + +static void crypto_user_direct_free_pages(struct crypto_user_direct_kern *k) +{ + kfree(k->sp); + kfree(k->dp); +} + +static int crypto_user_direct_alloc_vmas(struct crypto_user_direct_kern *k) +{ + k->svma = kmalloc(sizeof(struct vm_area_struct *) * k->snum, GFP_KERNEL); + if (!k->svma) { + dprintk("Failed to allocate %d source vmas.\n", k->snum); + return -ENOMEM; + } + + k->dvma = kmalloc(sizeof(struct vm_area_struct *) * k->dnum, GFP_KERNEL); + if (!k->dvma) { + dprintk("Failed to allocate %d destination vmas.\n", k->dnum); + kfree(k->svma); + return -ENOMEM; + } + + return 0; +} + +static void crypto_user_direct_free_vmas(struct crypto_user_direct_kern *k) +{ + kfree(k->svma); + kfree(k->dvma); +} + +static int crypto_user_direct_alloc_mm(struct crypto_user_direct_kern *k) +{ + int err; + + err = crypto_user_direct_alloc_pages(k); + if (err) + return err; + + err = crypto_user_direct_alloc_vmas(k); + if (err) { + crypto_user_direct_free_pages(k); + return err; + } + + return 0; +} + +static void crypto_user_direct_free_mm(struct crypto_user_direct_kern *k) +{ + crypto_user_direct_free_pages(k); + crypto_user_direct_free_vmas(k); +} + +static void crypto_user_direct_free_data(struct crypto_user_direct_kern *k) +{ + int i; + + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); + for (i=0; idnum; ++i) { + set_page_dirty_lock(k->dp[i]); + page_cache_release(k->dp[i]); + } + up_read(&k->mm->mmap_sem); + crypto_user_direct_free_mm(k); + mmput(k->mm); +} + +static void crypto_user_direct_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_user_direct_kern *k = data->priv; + struct cn_msg m; + + memset(&m, 0, sizeof(m)); + + memcpy(&m.id, &crypto_conn_id, sizeof(m.id)); + m.seq = k->seq; + m.ack = k->ack+1; + + cn_netlink_send(&m, 0, GFP_KERNEL); + + crypto_user_direct_free_data(k); + crypto_user_free_crypto_data(data); +} + +static void crypto_user_direct_fill_data(struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int i, size; + + size = k->usr.src_size; + for (i=0; isg_src_num; ++i) { + data->sg_src[i].page = k->sp[i]; + + if (i == 0) { + data->sg_src[i].offset = offset_in_page(k->usr.src); + data->sg_src[i].length = ALIGN_DATA_SIZE(k->usr.src) - k->usr.src; + } else { + data->sg_src[i].offset = 0; + data->sg_src[i].length = (i != data->sg_src_num)?PAGE_SIZE:size; + } + + size -= data->sg_src[i].length; + } + + size = k->usr.dst_size; + for (i=0; isg_dst_num; ++i) { + data->sg_dst[i].page = k->dp[i]; + + if (i == 0) { + data->sg_dst[i].offset = offset_in_page(k->usr.dst); + data->sg_dst[i].length = ALIGN_DATA_SIZE(k->usr.dst) - k->usr.dst; + } else { + data->sg_dst[i].offset = 0; + data->sg_dst[i].length = (i != data->sg_dst_num)?PAGE_SIZE:size; + } + + size -= data->sg_dst[i].length; + } +} + +static int crypto_user_direct_prepare_data(struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int err, i; + struct task_struct *tsk; + + tsk = find_task_by_pid(k->usr.pid); + if (!tsk) { + dprintk(KERN_ERR "Task with pid=%d does not exist.\n", k->usr.pid); + return -ENODEV; + } + + dprintk("Found task with pid=%d.\n", k->usr.pid); + + k->mm = get_task_mm(tsk); + if (!k->mm) + return -EINVAL; + + k->snum = data->sg_src_num; + k->dnum = data->sg_dst_num; + + err = crypto_user_direct_alloc_mm(k); + if (err) + goto err_out_put_mm; + + down_read(&k->mm->mmap_sem); + + err = get_user_pages(tsk, k->mm, k->usr.src, k->snum, 1, 1, k->sp, k->svma); + if (err < 0) { + dprintk("Failed to get %d src pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_up_sem; + } + + err = get_user_pages(tsk, k->mm, k->usr.dst, k->dnum, 1, 1, k->dp, k->dvma); + if (err < 0) { + dprintk("Failed to get %d dst pages for pid=%d.\n", + k->snum, k->usr.pid); + goto err_out_put_src; + } + + crypto_user_direct_fill_data(data, k); + + return 0; + +err_out_put_src: + for (i=0; isnum; ++i) + page_cache_release(k->sp[i]); +err_out_up_sem: + up_read(&k->mm->mmap_sem); + + crypto_user_direct_free_mm(k); +err_out_put_mm: + mmput(k->mm); + return err; +} + +static int crypto_user_direct_prepare(struct crypto_session_initializer *ci, struct crypto_data *data, struct crypto_user_direct_kern *k) +{ + int err; + + ci->operation = k->usr.operation; + ci->type = k->usr.type; + ci->mode = k->usr.mode; + ci->priority = k->usr.priority; + ci->callback = crypto_user_direct_callback; + + err = crypto_user_alloc_crypto_data(data, k->usr.src_size, k->usr.dst_size, k->usr.key_size, k->usr.iv_size); + if (err) + return err; + + if (k->usr.key_size) + crypto_user_fill_sg(k->key, k->usr.key_size, data->sg_key); + + if (k->usr.iv_size) + crypto_user_fill_sg(k->iv, k->usr.iv_size, data->sg_iv); + + data->priv = k; + data->priv_size = 0; + + err = crypto_user_direct_prepare_data(data, k); + if (err) { + crypto_user_free_crypto_data(data); + return err; + } + + return 0; +} + +static int crypto_user_direct_process(struct crypto_user_direct_kern *k) +{ + struct crypto_session_initializer ci; + struct crypto_data data; + struct crypto_session *s; + int err; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + err = crypto_user_direct_prepare(&ci, &data, k); + if (err) + return err; + + s = crypto_session_alloc(&ci, &data); + if (!s) { + crypto_user_direct_free_data(k); + return -EINVAL; + } + + return 0; +} + +static int crypto_user_direct_thread(void *data) +{ + struct crypto_user_direct_kern *k, *n; + + daemonize("crypto_user_direct_thread"); + allow_signal(SIGTERM); + + while (!need_exit) { + wait_event_interruptible_timeout(crypto_user_direct_wait_queue, + crypto_user_direct_has_work, 1000); + crypto_user_direct_has_work = 0; + + spin_lock_bh(&crypto_user_direct_lock); + list_for_each_entry_safe(k, n, &crypto_user_direct_list, entry) { + list_del(&k->entry); + + spin_unlock_bh(&crypto_user_direct_lock); + + crypto_user_direct_process(k); + + spin_lock_bh(&crypto_user_direct_lock); + } + spin_unlock_bh(&crypto_user_direct_lock); + } + + complete_and_exit(&thread_exited, 0); +} + +int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr) +{ + struct crypto_user_direct_kern *k; + + k = kmalloc(sizeof(struct crypto_user_direct_kern) + usr->key_size + usr->iv_size, GFP_ATOMIC); + if (!k) { + dprintk("Failed to allocate new kernel crypto request.\n"); + return -ENOMEM; + } + + memset(k, 0, sizeof(*k)); + + memcpy(&k->usr, usr, sizeof(k->usr)); + + k->key = (u8 *)(k+1); + k->iv = (u8 *)(k->key + k->usr.key_size); + + memcpy(k->key, usr->data, k->usr.key_size); + memcpy(k->iv, usr->data + k->usr.key_size, k->usr.iv_size); + + k->seq = seq; + k->ack = ack; + + spin_lock_bh(&crypto_user_direct_lock); + list_add(&k->entry, &crypto_user_direct_list); + spin_unlock_bh(&crypto_user_direct_lock); + + dprintk("msg [%08x.%08x]: op=[%04x.%04x.%04x.%04x], src=%llx [%d], dst=%llx [%d], key=%p [%d], iv=%p [%d].\n", + seq, ack, + k->usr.operation, k->usr.mode, k->usr.type, k->usr.priority, + k->usr.src, k->usr.src_size, + k->usr.dst, k->usr.dst_size, + k->key, k->usr.key_size, + k->iv, k->usr.iv_size); + + crypto_user_direct_has_work = 1; + wake_up_interruptible(&crypto_user_direct_wait_queue); + + return 0; +} + +int crypto_user_direct_init(void) +{ + long pid; + + init_completion(&thread_exited); + pid = kernel_thread(crypto_user_direct_thread, NULL, CLONE_FS | CLONE_FILES); + if (IS_ERR((void *)pid)) { + dprintk(KERN_ERR "Failed to create acrypto userspace processing thread.\n"); + return -EINVAL; + } + + printk(KERN_INFO "Acrypto userspace processing thread has been started.\n"); + + return 0; +} + +void crypto_user_direct_fini(void) +{ + need_exit = 1; + wait_for_completion(&thread_exited); + + printk(KERN_INFO "Acrypto userspace processing thread has been finished.\n"); +} diff --git a/acrypto/crypto_user_direct.h b/acrypto/crypto_user_direct.h new file mode 100644 index 0000000..2c94b0c --- /dev/null +++ b/acrypto/crypto_user_direct.h @@ -0,0 +1,74 @@ +/* + * crypto_user_direct.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_DIRECT_H +#define __CRYPTO_USER_DIRECT_H + +struct crypto_user_direct +{ + __u64 src; + __u32 src_size; + __u64 dst; + __u32 dst_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; + + int pid; + + int key_size; + int iv_size; + + __u8 data[0]; +}; + +#ifdef __KERNEL__ + +#include +#include +#include + +struct crypto_user_direct_kern +{ + struct list_head entry; + + u32 seq; + u32 ack; + + struct crypto_user_direct usr; + u8 *key; + u8 *iv; + + int snum, dnum; + struct page **sp, **dp; + struct vm_area_struct **svma, **dvma; + struct mm_struct *mm; +}; + +int crypto_user_direct_init(void); +void crypto_user_direct_fini(void); +int crypto_user_direct_add_request(u32 seq, u32 ack, struct crypto_user_direct *usr); + +#endif /* __KERNEL__ */ + +#endif /* __CRYPTO_USER_DIRECT_H */ diff --git a/acrypto/crypto_user_ioctl.c b/acrypto/crypto_user_ioctl.c new file mode 100644 index 0000000..9a86a94 --- /dev/null +++ b/acrypto/crypto_user_ioctl.c @@ -0,0 +1,263 @@ +/* + * crypto_user_ioctl.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "crypto_user.h" +#include "crypto_user_ioctl.h" + +static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg); +static int crypto_user_ioctl_open(struct inode *inode, struct file *fp); +static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp); +int crypto_user_ioctl_init(void); +void crypto_user_ioctl_fini(void); + +static int crypto_user_ioctl_major = 0; +static char crypto_user_ioctl_name[] = "crypto_user_ioctl"; + +static struct file_operations crypto_user_ioctl_ops = { + .open = crypto_user_ioctl_open, + .release = crypto_user_ioctl_release, + .ioctl = crypto_user_ioctl_ioctl, + .owner = THIS_MODULE +}; + +static void dump_data(u8 *ptr) +{ + int i; + + dprintk("USER DATA: "); + for (i=0; i<32; ++i) + dprintka("%02x ", ptr[i]); + dprintka("\n"); +} + +static int crypto_user_ioctl_open(struct inode *inode, struct file *fp) +{ + struct crypto_user_ioctl_kern *iok; + + iok = kmalloc(sizeof(*iok), GFP_KERNEL); + if (!iok) { + dprintk("Failed to allocate new crypto_user_ioctl_kern structure.\n"); + return -ENOMEM; + } + memset(iok, 0, sizeof(*iok)); + + fp->private_data = iok; + + return 0; +} + +static int crypto_user_ioctl_release(struct inode *pinode, struct file *fp) +{ + struct crypto_user_ioctl_kern *iok = fp->private_data; + int i; + + for (i=0; i<4; ++i) + if (iok->ptr[i]) + kfree(iok->ptr[i]); + kfree(iok); + + return 0; +} + +static void crypto_user_ioctl_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct crypto_user_ioctl_kern *iok = data->priv; + + dprintk("%s() for session %08x [%08x].\n", + __func__, iok->s->ci.id, iok->s->ci.dev_id); + + crypto_user_free_crypto_data(&iok->data); + + iok->scompleted = 1; + wake_up_interruptible(&iok->wait); +} + +static int crypto_user_ioctl_session_alloc(struct crypto_user_ioctl *io, struct crypto_user_ioctl_kern *iok) +{ + int err; + + err = crypto_user_alloc_crypto_data(&iok->data, io->src_size, io->dst_size, io->key_size, io->iv_size); + if (err) + return err; + + iok->ci.operation = io->operation; + iok->ci.type = io->type; + iok->ci.mode = io->mode; + iok->ci.priority = io->priority; + iok->ci.callback = crypto_user_ioctl_callback; + + iok->data.priv = iok; + iok->data.priv_size = 0; + + iok->scompleted = 0; + + init_waitqueue_head(&iok->wait); + + iok->s = crypto_session_create(&iok->ci, &iok->data); + if (!iok->s) { + crypto_user_free_crypto_data(&iok->data); + return -ENODEV; + } + + return 0; +} + +static int crypto_user_ioctl_session_add(struct crypto_user_ioctl_kern *iok) +{ + crypto_session_add(iok->s); + + return 0; +} + +static int crypto_user_ioctl_ioctl(struct inode *inode, struct file *fp, unsigned int cmd, unsigned long arg) +{ + struct crypto_user_ioctl io; + struct crypto_user_data data; + unsigned long not_read; + int err; + struct crypto_user_ioctl_kern *iok; + + iok = fp->private_data; + + err = 0; + switch (cmd) { + case CRYPTO_SESSION_ALLOC: + not_read = copy_from_user(&io, (void __user *)arg, sizeof(io)); + if (not_read) { + dprintk("Failed to read crypto_user_ioctl structure from userspace.\n"); + err = -EINVAL; + break; + } + + err = crypto_user_ioctl_session_alloc(&io, iok); + break; + case CRYPTO_FILL_DATA: + not_read = copy_from_user(&data, (void __user *)arg, sizeof(data)); + if (not_read) { + dprintk("Failed to read crypto_user_ioctl_data structure from userspace.\n"); + err = -EINVAL; + break; + } + + if (data.data_size > MAX_DATA_SIZE * PAGE_SIZE) { + dprintk("Data size is too bit: size=%u, type=%x.\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (!crypto_user_get_sg(&data, &iok->data)) { + dprintk("Invalid crypto_user_data structure [size=%u, type=%x].\n", + data.data_size, data.data_type); + err = -EINVAL; + break; + } + + if (iok->ptr[data.data_type]) + kfree(iok->ptr[data.data_type]); + + iok->ptr[data.data_type] = kmalloc(data.data_size, GFP_KERNEL); + if (!iok->ptr[data.data_type]) { + dprintk("Failed to allocate %d bytes for data type %d.\n", + data.data_size, data.data_type); + err = -ENOMEM; + break; + } + + not_read = copy_from_user(iok->ptr[data.data_type], (void __user *)arg + sizeof(data), data.data_size); + if (not_read) { + dprintk("Failed to read %d bytes of crypto data [type=%d] from userspace.\n", + data.data_size, data.data_type); + kfree(iok->ptr[data.data_type]); + err = -EINVAL; + break; + } + + memcpy(&iok->usr[data.data_type], &data, sizeof(struct crypto_user_data)); + + err = crypto_user_fill_sg_data(&data, &iok->data, iok->ptr[data.data_type]); + if (err) { + kfree(iok->ptr[data.data_type]); + break; + } + break; + case CRYPTO_SESSION_ADD: + if (!iok->s) { + dprintk("CRYPTO_SESSION_ADD must be called after session initialisation.\n"); + err = -EINVAL; + break; + } + + err = crypto_user_ioctl_session_add(iok); + if (err) + break; + + wait_event_interruptible(iok->wait, iok->scompleted); + + dump_data(iok->ptr[CRYPTO_USER_DATA_DST]); + + not_read = copy_to_user((void __user *)arg, iok->ptr[CRYPTO_USER_DATA_DST], iok->usr[CRYPTO_USER_DATA_DST].data_size); + if (not_read) { + dprintk("Failed to copy to user %d bytes of result.\n", iok->usr[CRYPTO_USER_DATA_DST].data_size); + err = -EINVAL; + break; + } + break; + + default: + dprintk("Invalid ioctl(0x%x).\n", cmd); + err = -ENODEV; + break; + } + + return err; +} + +int crypto_user_ioctl_init(void) +{ + crypto_user_ioctl_major = register_chrdev(0, crypto_user_ioctl_name, &crypto_user_ioctl_ops); + if (crypto_user_ioctl_major < 0) { + dprintk("Failed to register %s char device: err=%d.\n", crypto_user_ioctl_name, crypto_user_ioctl_major); + return -ENODEV; + }; + + printk("Asynchronous crypto userspace helper(ioctl based) has been started, major=%d.\n", crypto_user_ioctl_major); + + return 0; +} + +void crypto_user_ioctl_fini(void) +{ + unregister_chrdev(crypto_user_ioctl_major, crypto_user_ioctl_name); +} diff --git a/acrypto/crypto_user_ioctl.h b/acrypto/crypto_user_ioctl.h new file mode 100644 index 0000000..b2dfcd0 --- /dev/null +++ b/acrypto/crypto_user_ioctl.h @@ -0,0 +1,67 @@ +/* + * crypto_user_ioctl.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_USER_IOCTL_H +#define __CRYPTO_USER_IOCTL_H + +struct crypto_user_ioctl +{ + __u16 src_size; + __u16 dst_size; + __u16 key_size; + __u16 iv_size; + + __u16 operation; + __u16 type; + __u16 mode; + __u16 priority; +}; + +#define CRYPTO_USER_IOCTL_SYM 'U' +#define CRYPTO_SESSION_ALLOC _IOW(CRYPTO_USER_IOCTL_SYM, 0, struct crypto_user_ioctl) +#define CRYPTO_SESSION_ADD _IOR(CRYPTO_USER_IOCTL_SYM, 1, char *) +#define CRYPTO_FILL_DATA _IOW(CRYPTO_USER_IOCTL_SYM, 2, struct crypto_user_data) + + +#ifdef __KERNEL__ + +#include + +#include "crypto_user.h" + +struct crypto_user_ioctl_kern +{ + struct crypto_session_initializer ci; + struct crypto_data data; + struct crypto_session *s; + + int scompleted; + wait_queue_head_t wait; + + struct crypto_user_data usr[4]; + void *ptr[4]; +}; + +int crypto_user_ioctl_init(void); +void crypto_user_ioctl_fini(void); + +#endif /* __KERNEL__ */ +#endif /* __CRYPTO_USER_IOCTL_H */ diff --git a/acrypto/simple_lb.c b/acrypto/simple_lb.c new file mode 100644 index 0000000..5b97f82 --- /dev/null +++ b/acrypto/simple_lb.c @@ -0,0 +1,69 @@ +/* + * simple_lb.c + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include + +#include "crypto_lb.h" + +extern struct list_head cdev_list; +extern spinlock_t cdev_lock; + +static struct crypto_device *simple_lb_find_device(struct crypto_lb *, + struct crypto_session_initializer *, + struct crypto_data *); + +struct crypto_lb simple_lb = { + .name = "simple_lb", + .find_device = simple_lb_find_device +}; + +static struct crypto_device *simple_lb_find_device(struct crypto_lb *lb, + struct crypto_session_initializer *ci, + struct crypto_data *data) +{ + struct crypto_device *dev, *__dev = NULL; + int min = 0x7ffffff; + unsigned long flags; + + spin_lock_irqsave(&cdev_lock, flags); + list_for_each_entry(dev, &cdev_list, cdev_entry) { + if (device_broken(dev)) + continue; + if (!match_initializer(dev, ci)) + continue; + + if (atomic_read(&dev->refcnt) < min) { + min = atomic_read(&dev->refcnt); + __dev = dev; + } + } + if (__dev) + crypto_device_get(__dev); + spin_unlock_irqrestore(&cdev_lock, flags); + + return __dev; +} + diff --git a/arch/i386/Kconfig b/arch/i386/Kconfig index 6004bb0..2357d90 100644 --- a/arch/i386/Kconfig +++ b/arch/i386/Kconfig @@ -1018,6 +1018,8 @@ source "security/Kconfig" source "crypto/Kconfig" +source "acrypto/Kconfig" + source "lib/Kconfig" # diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig index 6ece645..173abd2 100644 --- a/arch/x86_64/Kconfig +++ b/arch/x86_64/Kconfig @@ -578,4 +578,6 @@ source "security/Kconfig" source "crypto/Kconfig" +source "acrypto/Kconfig" + source "lib/Kconfig" diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index cf66310..5b30e7f 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -22,6 +22,24 @@ #define PFX "crypt: " +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE +#include +#include + +struct dm_async_io +{ + atomic_t pending; + int write; + struct completion complete; +}; + +struct crypt_io; +struct dm_async_priv +{ + struct crypt_io *io; +}; +#endif + /* * per bio private data */ @@ -32,6 +50,9 @@ struct crypt_io { struct work_struct work; atomic_t pending; int error; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + struct dm_async_io aio; +#endif }; /* @@ -227,11 +248,192 @@ static struct crypt_iv_operations crypt_ .generator = crypt_iv_essiv_gen }; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + +static mempool_t *dm_acrypto_pool; +static kmem_cache_t *dm_acrypto_cache; +static int dm_acrypto_size = sizeof(struct dm_async_priv) + 4*sizeof(struct scatterlist) + 128*2; +static int dm_acrypto_use = 1; +module_param(dm_acrypto_use, uint, 0); + +static inline u16 crypto_tfm_get_type(struct crypto_tfm *tfm) +{ + u16 type; + char *name; + + name = (char *)crypto_tfm_alg_name(tfm); + + if (!strncmp(name, "aes", 3)) { + switch (crypto_tfm_alg_blocksize(tfm) << 3) { + case 128: + type = CRYPTO_TYPE_AES_128; + break; + case 192: + type = CRYPTO_TYPE_AES_192; + break; + case 256: + type = CRYPTO_TYPE_AES_256; + break; + default: + type = 0xffff; + break; + } + } else if (!strncmp(name, "des3_ede", 3)) + type = CRYPTO_TYPE_3DES; + else + type = 0xffff; + + return type; +} + +static inline u16 crypto_tfm_get_mode(struct crypto_tfm *tfm) +{ + u16 mode = tfm->crt_cipher.cit_mode & CRYPTO_TFM_MODE_MASK; + + switch (mode) { + case CRYPTO_TFM_MODE_ECB: + mode = CRYPTO_MODE_ECB; + break; + case CRYPTO_TFM_MODE_CBC: + mode = CRYPTO_MODE_CBC; + break; + case CRYPTO_TFM_MODE_CFB: + mode = CRYPTO_MODE_CFB; + break; + case CRYPTO_TFM_MODE_CTR: + mode = CRYPTO_MODE_CTR; + break; + default: + mode = 0xffff; + break; + } + + return mode; +} + +static void dec_pending(struct crypt_io *io, int error); + +static inline void dm_finish(struct crypt_io *io, int error) +{ + if (atomic_dec_and_test(&io->aio.pending)) { + if (io->aio.write) + complete(&io->aio.complete); + else + dec_pending(io, error); + } +} + +static void dm_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct dm_async_priv *priv = data->priv; + + BUG_ON(!priv); + BUG_ON(!priv->io); + + dm_finish(priv->io, 0); + + mempool_free(priv, dm_acrypto_pool); +} + +static inline int acrypto_process(struct crypt_config *cc, struct crypt_io *io, struct scatterlist *out, + struct scatterlist *in, unsigned int len, u8 *iv, int iv_size, int pos) +{ + struct crypto_session *s; + struct crypto_session_initializer ci; + struct crypto_data data; + struct scatterlist *sg; + int err = 0; + u8 *local_key; + struct dm_async_priv *priv; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + ci.operation = (io->aio.write)?CRYPTO_OP_ENCRYPT:CRYPTO_OP_DECRYPT; + ci.priority = 0; + ci.callback = &dm_callback; + ci.type = crypto_tfm_get_type(cc->tfm); + ci.mode = crypto_tfm_get_mode(cc->tfm); + + if (ci.type == 0xffff || ci.mode == 0xffff) { + err = -EINVAL; + goto err_out_exit; + } + + data.sg_src_num = data.sg_dst_num = data.sg_key_num = 1; + data.sg_iv_num = (iv)?1:0; + + priv = mempool_alloc(dm_acrypto_pool, GFP_KERNEL); + if (!priv) { + err = -ENOMEM; + goto err_out_exit; + } + + priv->io = io; + + data.priv = priv; + + sg = (struct scatterlist *)(priv + 1); + + data.sg_src = &sg[0]; + data.sg_dst = &sg[1]; + + data.sg_src[0].page = in->page; + data.sg_src[0].offset = in->offset; + data.sg_src[0].length = in->length; + + data.sg_dst[0].page = out->page; + data.sg_dst[0].offset = out->offset; + data.sg_dst[0].length = out->length; + + data.sg_key = &sg[2]; + local_key = (u8 *)&sg[3]; + + memcpy(local_key, cc->key, cc->key_size); + + data.sg_key[0].page = virt_to_page(local_key); + data.sg_key[0].offset = offset_in_page(local_key); + data.sg_key[0].length = cc->key_size; + + if (iv) { + u8 *local_iv; + + data.sg_iv = (struct scatterlist *)(local_key + cc->key_size); + local_iv = (u8 *)(local_key + cc->key_size + sizeof(*sg)); + + data.sg_iv[0].page = virt_to_page(local_iv); + data.sg_iv[0].offset = offset_in_page(local_iv); + data.sg_iv[0].length = iv_size; + + memcpy(local_iv, iv, iv_size); + } + + s = crypto_session_alloc(&ci, &data); + if (!s) { + err = -ENOMEM; + goto err_out_free; + } + + return 0; + +err_out_free: + mempool_free(priv, dm_acrypto_pool); +err_out_exit: + + return err; +} +#else +static inline int acrypto_process(struct crypt_config *cc, struct crypt_io *io, struct scatterlist *out, + struct scatterlist *in, unsigned int len, u8 *iv, int iv_size, int write, int pos) +{ + return 0; +} +#endif static inline int crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out, struct scatterlist *in, unsigned int length, - int write, sector_t sector) + int write, sector_t sector, struct crypt_io *io, int pos) { u8 iv[cc->iv_size]; int r; @@ -241,11 +443,19 @@ crypt_convert_scatterlist(struct crypt_c if (r < 0) return r; + r = acrypto_process(cc, io, out, in, length, iv, cc->iv_size, pos); + if (r == 0) + return r; + if (write) r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv); else r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv); } else { + r = acrypto_process(cc, io, out, in, length, NULL, 0, pos); + if (r == 0) + return r; + if (write) r = crypto_cipher_encrypt(cc->tfm, out, in, length); else @@ -274,9 +484,50 @@ crypt_convert_init(struct crypt_config * * Encrypt / decrypt data from one bio to another one (can be the same one) */ static int crypt_convert(struct crypt_config *cc, - struct convert_context *ctx) + struct convert_context *ctx, + struct crypt_io *io) { int r = 0; + sector_t first_sector = ctx->sector; + +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + int num = 0; + + if (dm_acrypto_use) { + unsigned int idx_in, idx_out, offset_in, offset_out; + + idx_in = ctx->idx_in; + idx_out = ctx->idx_out; + offset_in = ctx->offset_in; + offset_out = ctx->offset_out; + + while(idx_in < ctx->bio_in->bi_vcnt && + idx_out < ctx->bio_out->bi_vcnt) { + + struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, idx_in); + struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, idx_out); + + offset_in += 1 << SECTOR_SHIFT; + offset_out += 1 << SECTOR_SHIFT; + + if (offset_in >= bv_in->bv_len) { + offset_in = 0; + idx_in++; + } + + if (offset_out >= bv_out->bv_len) { + offset_out = 0; + idx_out++; + } + + num++; + } + + init_completion(&io->aio.complete); + atomic_set(&io->aio.pending, num); + io->aio.write = ctx->write; + } +#endif while(ctx->idx_in < ctx->bio_in->bi_vcnt && ctx->idx_out < ctx->bio_out->bi_vcnt) { @@ -306,13 +557,26 @@ static int crypt_convert(struct crypt_co } r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length, - ctx->write, ctx->sector); + ctx->write, ctx->sector, io, + ctx->sector - first_sector); if (r < 0) break; - ctx->sector++; } + /* + * If there is an IO here, then this request was done through crypt_endio() + * and queued work and we can simply return from this request. + * If there is no IO here, then it was bulk of requests from crypt_map() + * and we can not just return, since crypt_map() is designed + * to be synchronous. + */ + if (!ctx->write && r) + dec_pending(io, r); + + if (ctx->write) + wait_for_completion(&io->aio.complete); + return r; } @@ -461,13 +725,10 @@ static void kcryptd_do_work(void *data) struct crypt_io *io = (struct crypt_io *) data; struct crypt_config *cc = (struct crypt_config *) io->target->private; struct convert_context ctx; - int r; crypt_convert_init(cc, &ctx, io->bio, io->bio, io->bio->bi_sector - io->target->begin, 0); - r = crypt_convert(cc, &ctx); - - dec_pending(io, r); + crypt_convert(cc, &ctx, io); } static void kcryptd_queue_io(struct crypt_io *io) @@ -752,7 +1013,7 @@ crypt_clone(struct crypt_config *cc, str io->first_clone, bvec_idx); if (clone) { ctx->bio_out = clone; - if (crypt_convert(cc, ctx) < 0) { + if (crypt_convert(cc, ctx, io) < 0) { crypt_free_buffer_pages(cc, clone, clone->bi_size); bio_put(clone); @@ -916,6 +1177,25 @@ static struct target_type crypt_target = static int __init dm_crypt_init(void) { int r; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + printk("%s: Compiled with ACRYPTO support.\n", __func__); + + dm_acrypto_cache = kmem_cache_create("dm_acrypto_io", dm_acrypto_size, 0, 0, NULL, NULL); + if (!dm_acrypto_cache) { + dm_acrypto_use = 0; + goto sync_init; + } + + dm_acrypto_pool = mempool_create(MIN_IOS, mempool_alloc_slab, + mempool_free_slab, dm_acrypto_cache); + if (!dm_acrypto_pool) { + kmem_cache_destroy(dm_acrypto_cache); + dm_acrypto_use = 0; + goto sync_init; + } + +sync_init: +#endif _crypt_io_pool = kmem_cache_create("dm-crypt_io", sizeof(struct crypt_io), @@ -942,6 +1222,12 @@ bad2: destroy_workqueue(_kcryptd_workqueue); bad1: kmem_cache_destroy(_crypt_io_pool); +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + if (dm_acrypto_use) { + mempool_destroy(dm_acrypto_pool); + kmem_cache_destroy(dm_acrypto_cache); + } +#endif return r; } @@ -954,6 +1240,12 @@ static void __exit dm_crypt_exit(void) destroy_workqueue(_kcryptd_workqueue); kmem_cache_destroy(_crypt_io_pool); +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + if (dm_acrypto_use) { + mempool_destroy(dm_acrypto_pool); + kmem_cache_destroy(dm_acrypto_cache); + } +#endif } module_init(dm_crypt_init); diff --git a/include/linux/acrypto.h b/include/linux/acrypto.h new file mode 100644 index 0000000..a6be34c --- /dev/null +++ b/include/linux/acrypto.h @@ -0,0 +1,245 @@ +/* + * acrypto.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __ACRYPTO_H +#define __ACRYPTO_H + +#define SCACHE_NAMELEN 32 + +struct crypto_session_initializer; +struct crypto_data; +typedef void (*crypto_callback_t) (struct crypto_session_initializer *, + struct crypto_data *); + +struct crypto_device_stat +{ + __u64 scompleted; + __u64 sfinished; + __u64 sstarted; + __u64 kmem_failed; + __u64 route_failed; + __u64 pool_failed; +}; + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include +#include +#include + +#include + +//#define DEBUG +#ifdef DEBUG +#define dprintk(f, a...) printk(f, ##a) +#define dprintka(f, a...) printk(f, ##a) +#else +#define dprintk(f, a...) +#define dprintka(f, a...) +#endif + +#define SESSION_COMPLETED 15 +#define SESSION_DIRECT 12 +#define SESSION_BOUND 11 +#define SESSION_BROKEN 10 +#define SESSION_FROM_CACHE 9 + +#define session_completed(s) test_bit(SESSION_COMPLETED, &s->ci.flags) +struct crypto_session; +extern int complete_session(struct crypto_session *s); +#define uncomplete_session(s) clear_bit(SESSION_COMPLETED, &s->ci.flags) + +#define session_bound(s) test_bit(SESSION_BOUND, &s->ci.flags) +#define bind_session(s) set_bit(SESSION_BOUND, &s->ci.flags) +#define unbind_session(s) clear_bit(SESSION_BOUND, &s->ci.flags) +#define sci_bound(ci) test_bit(SESSION_BOUND, &ci->flags) + +#define session_broken(s) test_bit(SESSION_BROKEN, &s->ci.flags) +#define break_session(s) set_bit(SESSION_BROKEN, &s->ci.flags) +#define unbreak_session(s) clear_bit(SESSION_BROKEN, &s->ci.flags) + +#define session_from_cache(s) test_bit(SESSION_FROM_CACHE, &s->ci.flags) +#define mark_session_from_cache(s) set_bit(SESSION_FROM_CACHE, &s->ci.flags) + +#define CRYPTO_MAX_PRIV_SIZE 1024 + +#define DEVICE_BROKEN 0 + +#define device_broken(dev) test_bit(DEVICE_BROKEN, &dev->flags) +#define break_device(dev) set_bit(DEVICE_BROKEN, &dev->flags) +#define repair_device(dev) clear_bit(DEVICE_BROKEN, &dev->flags) + +struct crypto_capability { + u16 operation; + u16 type; + u16 mode; + u16 qlen; + u64 ptime; + u64 scomp; +}; + +struct crypto_session_initializer { + u16 operation; + u16 type; + u16 mode; + u16 priority; + + u32 id; + u32 dev_id; + + long flags; + + u32 bdev; + + u64 ptime; + + crypto_callback_t callback; +}; + +struct crypto_data { + struct scatterlist *sg_src; + int sg_src_num; + struct scatterlist *sg_dst; + int sg_dst_num; + struct scatterlist *sg_key; + int sg_key_num; + struct scatterlist *sg_iv; + int sg_iv_num; + + void *priv; + unsigned int priv_size; +}; + +struct crypto_device { + char name[SCACHE_NAMELEN]; + char route_cache_name[SCACHE_NAMELEN]; + + spinlock_t session_lock; + struct list_head session_list; + + atomic_t sid; + spinlock_t lock; + + atomic_t refcnt; + + long flags; + + u32 id; + + struct list_head cdev_entry; + + void (*data_ready)(struct crypto_device *); + + struct device_driver *driver; + struct device device; + struct class_device class_device; + struct completion dev_released; + + spinlock_t stat_lock; + struct crypto_device_stat stat; + + struct crypto_capability *cap; + int cap_number; + + void *priv; + + mempool_t *session_pool; + kmem_cache_t *session_cache; + + mempool_t *route_pool; + kmem_cache_t *route_cache; +}; + +struct crypto_route_head { + struct crypto_route *next; + struct crypto_route *prev; + + u32 qlen; + spinlock_t lock; +}; + +struct crypto_route { + struct crypto_route *next; + struct crypto_route *prev; + + struct crypto_route_head *list; + struct crypto_device *dev; + + struct crypto_session_initializer ci; +}; + +struct crypto_session { + struct list_head dev_queue_entry; + + struct crypto_session_initializer ci; + + struct crypto_data data; + + spinlock_t lock; + + struct work_struct work; + + struct crypto_route_head route_list; + + struct crypto_device *pool_dev; +}; + +struct crypto_session *crypto_session_alloc(struct crypto_session_initializer *, struct crypto_data *); +struct crypto_session *crypto_session_create(struct crypto_session_initializer *, struct crypto_data *); +void crypto_session_destroy(struct crypto_session *); +int __crypto_session_add(struct crypto_device *, struct crypto_session *); +int crypto_session_add(struct crypto_session *); + +int crypto_session_dequeue_route(struct crypto_session *); + +void crypto_device_get(struct crypto_device *); +void crypto_device_put(struct crypto_device *); +struct crypto_device *crypto_device_get_name(char *); + +int __crypto_device_add(struct crypto_device *); +int crypto_device_add(struct crypto_device *); +void __crypto_device_remove(struct crypto_device *); +void crypto_device_remove(struct crypto_device *); +int match_initializer(struct crypto_device *, struct crypto_session_initializer *); +int __match_initializer(struct crypto_capability *, struct crypto_session_initializer *, struct crypto_device *); + +int __crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s); +int crypto_session_enqueue(struct crypto_device *dev, struct crypto_session *s); +struct crypto_session *crypto_session_dequeue(struct crypto_device *dev); + +static inline int compare_device(struct crypto_device *d1, struct crypto_device *d2) +{ + return (d1->id == d2->id)?1:0; +} + +void crypto_process(struct crypto_session *s, + int (*proc)(struct crypto_session *s, void *src, int slen, void *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + int (*proc_scatter)(struct crypto_session *s, struct scatterlist *src, int slen, + struct scatterlist *dst, int dlen, void *key, int klen, void *iv, int ilen, void *data), + void (*finish)(struct crypto_session *s, void *data), + void *data); + +#endif /* __KERNEL__ */ +#endif /* __ACRYPTO_H */ diff --git a/include/linux/crypto_def.h b/include/linux/crypto_def.h new file mode 100644 index 0000000..b701b1e --- /dev/null +++ b/include/linux/crypto_def.h @@ -0,0 +1,41 @@ +/* + * crypto_def.h + * + * Copyright (c) 2004 Evgeniy Polyakov + * + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef __CRYPTO_DEF_H +#define __CRYPTO_DEF_H + +#define CRYPTO_OP_DECRYPT 0 +#define CRYPTO_OP_ENCRYPT 1 +#define CRYPTO_OP_HMAC 2 +#define CRYPTO_OP_RNG 3 + +#define CRYPTO_MODE_ECB 0 +#define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_CFB 2 +#define CRYPTO_MODE_OFB 3 +#define CRYPTO_MODE_CTR 4 + +#define CRYPTO_TYPE_AES_128 0 +#define CRYPTO_TYPE_AES_192 1 +#define CRYPTO_TYPE_AES_256 2 +#define CRYPTO_TYPE_3DES 3 + +#endif /* __CRYPTO_DEF_H */ diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 1cdb879..40ab7be 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -82,6 +82,8 @@ extern struct semaphore xfrm_cfg_sem; metrics. Plus, it will be made via sk->sk_dst_cache. Solved. */ +struct xfrm4_input_shared; + /* Full description of state of transformer. */ struct xfrm_state { @@ -147,6 +149,9 @@ struct xfrm_state /* Private data of this transformer, format is opaque, * interpreted by xfrm_type methods. */ void *data; + + /* Used to store shared XFRM callback entry */ + struct xfrm4_input_shared *shared; }; enum { @@ -566,6 +571,31 @@ struct sec_path struct sec_decap_state x[XFRM_MAX_DEPTH]; }; +struct xfrm4_input_shared +{ + struct sk_buff *skb; + int xfrm_nr, first, xfrm_encap; + struct sec_decap_state xfrm_vec[XFRM_MAX_DEPTH]; + __u16 encap_type; + int decaps; + u32 seq, spi; + int (*callback)(struct xfrm4_input_shared *sh); + atomic_t refcnt; + u8 transform[256]; +}; + +static inline void xfrm_shared_get(struct xfrm4_input_shared *sh) +{ + atomic_inc(&sh->refcnt); +} + +static inline void xfrm_shared_put(struct xfrm4_input_shared *sh) +{ + if (atomic_dec_and_test(&sh->refcnt)) { + kfree(sh); + } +} + static inline struct sec_path * secpath_get(struct sec_path *sp) { diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 1b18ce6..ae79228 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -8,8 +8,13 @@ #include #include #include +#include #include #include +#include +#include +#include +#include /* decapsulation data for use when post-processing */ struct esp_decap_data { @@ -18,6 +23,166 @@ struct esp_decap_data { __u8 proto; }; +static int esp_output(struct xfrm_state *x, struct sk_buff *skb); + +struct esp_async { + struct sk_buff *skb; + struct dst_entry *dst; + struct sk_buff *trailer; + int clen; + u8 transform[256]; +}; + +static void esp4_callback(unsigned long data) +{ + struct esp_async *ea = (struct esp_async *)data; + struct sk_buff *skb = ea->skb; + struct dst_entry *dst = ea->dst; + struct esp_data *esp = (struct esp_data *)(ea->transform); + struct iphdr *top_iph; + struct ip_esp_hdr *esph; + struct crypto_tfm *tfm = esp->conf.tfm; + int alen, blksize, err; + + top_iph = skb->nh.iph; + esph = (struct ip_esp_hdr *)(skb->nh.raw + top_iph->ihl*4); + alen = esp->auth.icv_trunc_len; + blksize = (crypto_tfm_alg_blocksize(tfm) + 3) & ~3; + + if (esp->conf.ivlen) { + memcpy(esph->enc_data, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); + crypto_cipher_get_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); + } + + if (esp->auth.icv_full_len) { + esp->auth.icv(esp, skb, (u8*)esph-skb->data, + sizeof(struct ip_esp_hdr) + esp->conf.ivlen+ea->clen, ea->trailer->tail); + pskb_put(skb, ea->trailer, alen); + } + + ip_send_check(top_iph); + + kfree_skb(ea->trailer); + + skb->dst = dst_pop(dst); + if (!skb->dst) + goto err_out; + + err = dst_output(skb); + if (err) + goto err_out; + +out: + kfree(ea); + return; + +err_out: + kfree_skb(skb); + goto out; +} + + +static struct esp_async *esp_output_async_prepare(struct xfrm_state *x, struct sk_buff *skb, struct sk_buff *trailer, int clen) +{ + struct esp_async *ea; + + ea = kzalloc(sizeof(*ea), GFP_ATOMIC); + if (!ea) + return NULL; + + ea->skb = skb_clone(skb, GFP_ATOMIC); + if (!ea->skb) + goto err_out_free_ea; + + /* + * Oh, crap, actually we could use some reference counter magic like in skb_cow_data(); + */ + ea->trailer = skb_clone(trailer, GFP_ATOMIC); + if (!ea->trailer) + goto err_out_free_skb; + + ea->dst = skb->dst; + ea->clen = clen; + memcpy(ea->transform, x->data, sizeof(struct esp_data)); + + kfree_skb(skb); + + return ea; + +err_out_free_skb: + kfree(ea->skb); + +err_out_free_ea: + kfree(ea); + + return NULL; +} + +static inline u16 crypto_tfm_get_mode(struct crypto_tfm *tfm) +{ + u16 mode = tfm->crt_cipher.cit_mode & CRYPTO_TFM_MODE_MASK; + + switch (mode) { + case CRYPTO_TFM_MODE_ECB: + mode = CRYPTO_MODE_ECB; + break; + case CRYPTO_TFM_MODE_CBC: + mode = CRYPTO_MODE_CBC; + break; + case CRYPTO_TFM_MODE_CFB: + mode = CRYPTO_MODE_CFB; + break; + case CRYPTO_TFM_MODE_CTR: + mode = CRYPTO_MODE_CTR; + break; + default: + mode = 0xffff; + break; + } + + return mode; +} + +static inline u16 crypto_tfm_get_type(struct crypto_tfm *tfm) +{ + u16 type; + char *name; + + name = (char *)crypto_tfm_alg_name(tfm); + + if (!strncmp(name, "aes", 3)) { + switch (crypto_tfm_alg_blocksize(tfm) << 3) { + case 128: + type = CRYPTO_TYPE_AES_128; + break; + case 192: + type = CRYPTO_TYPE_AES_192; + break; + case 256: + type = CRYPTO_TYPE_AES_256; + break; + default: + type = 0xffff; + break; + } + } else if (!strncmp(name, "des3_ede", 3)) + type = CRYPTO_TYPE_3DES; + else + type = 0xffff; + + return type; +} + +static void esp4_async_output_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + void *key = page_address(data->sg_key[0].page) + data->sg_key[0].offset; + + esp4_callback((unsigned long)data->priv); + + kfree(key); + kfree(data->sg_src); +} + static int esp_output(struct xfrm_state *x, struct sk_buff *skb) { int err; @@ -30,6 +195,13 @@ static int esp_output(struct xfrm_state int clen; int alen; int nfrags; +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + struct crypto_session_initializer ci; + struct crypto_data data; + struct scatterlist *sg; + struct crypto_session *s; + u8 *key, *iv; +#endif /* Strip IP+ESP header. */ __skb_pull(skb, skb->h.raw - skb->data); @@ -38,6 +210,7 @@ static int esp_output(struct xfrm_state err = -ENOMEM; /* Round to block size */ + clen = skb->len; esp = x->data; @@ -97,6 +270,81 @@ static int esp_output(struct xfrm_state esph->spi = x->id.spi; esph->seq_no = htonl(++x->replay.oseq); +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + nfrags++; /* key */ + + if (esp->conf.ivlen) + nfrags++; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + ci.operation = CRYPTO_OP_ENCRYPT; + ci.mode = crypto_tfm_get_mode(tfm); + ci.type = crypto_tfm_get_type(tfm); + ci.priority = 0; + ci.callback = &esp4_async_output_callback; + + if (ci.mode == 0xffff || ci.type == 0xffff) + goto sync_crypto; + + sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC); + if (!sg) + goto error; + skb_to_sgvec(skb, sg, esph->enc_data+esp->conf.ivlen-skb->data, clen); + data.sg_src = data.sg_dst = sg; + + key = kmalloc(crypto_tfm_alg_ivsize(tfm) + esp->conf.key_len, GFP_ATOMIC); + if (!key) + goto err_out_free_sg; + + iv = key + esp->conf.key_len; + + if (esp->conf.ivlen) { + data.sg_key = &sg[nfrags - 2]; + data.sg_iv = &sg[nfrags - 1]; + data.sg_key_num = data.sg_iv_num = 1; + } else { + data.sg_key = &sg[nfrags - 1]; + data.sg_iv = NULL; + data.sg_key_num = 1; + data.sg_iv_num = 0; + } + + data.sg_src_num = data.sg_dst_num = nfrags - data.sg_key_num - data.sg_iv_num; + + memcpy(key, esp->conf.key, esp->conf.key_len); + data.sg_key[0].offset = offset_in_page(key); + data.sg_key[0].length = esp->conf.key_len; + data.sg_key[0].page = virt_to_page(key); + + if (esp->conf.ivlen) { + memcpy(iv, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); + data.sg_iv[0].offset = offset_in_page(iv); + data.sg_iv[0].length = crypto_tfm_alg_ivsize(tfm); + data.sg_iv[0].page = virt_to_page(iv); + } + + data.priv = esp_output_async_prepare(x, skb, trailer, clen); + if (!data.priv) + goto err_out_free_key; + + s = crypto_session_alloc(&ci, &data); + if (!s) + goto err_out_free_ea; + + return 0; + +err_out_free_ea: + kfree(data.priv); +err_out_free_key: + kfree(key); +err_out_free_sg: + kfree(sg); + goto sync_crypto; + +sync_crypto: +#endif if (esp->conf.ivlen) crypto_cipher_set_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); @@ -113,7 +361,7 @@ static int esp_output(struct xfrm_state if (unlikely(sg != &esp->sgbuf[0])) kfree(sg); } while (0); - + if (esp->conf.ivlen) { memcpy(esph->enc_data, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); crypto_cipher_get_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm)); @@ -126,13 +374,271 @@ static int esp_output(struct xfrm_state } ip_send_check(top_iph); - + err = 0; error: return err; } +static int esp4_sync_input_process(struct sk_buff *skb, struct esp_data *esp, + struct xfrm_decap_state *decap, int nfrags, int alen, int elen, int async, + int xfrm_encap) +{ + u8 nexthdr[2]; + struct scatterlist *sg = &esp->sgbuf[0]; + u8 workbuf[60]; + int padlen, encap_len = 0; + struct iphdr *iph = skb->nh.iph; + struct ip_esp_hdr *esph = (struct ip_esp_hdr*)skb->data; + int err = 0; + + if (!async) { + if (unlikely(nfrags > ESP_NUM_FAST_SG)) { + sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC); + if (!sg) { + err = -ENOMEM; + goto out; + } + } + skb_to_sgvec(skb, sg, sizeof(struct ip_esp_hdr) + esp->conf.ivlen, elen); + + crypto_cipher_decrypt(esp->conf.tfm, sg, sg, elen); + + if (unlikely(sg != &esp->sgbuf[0])) + kfree(sg); + } + + if (skb_copy_bits(skb, skb->len-alen-2, nexthdr, 2)) + BUG(); + + padlen = nexthdr[0]; + if (padlen+2 >= elen) { + err = -EINVAL; + goto out; + } + + /* ... check padding bits here. Silly. :-) */ + + if (xfrm_encap && decap && decap->decap_type) { + struct esp_decap_data *encap_data; + struct udphdr *uh = (struct udphdr *) (iph+1); + + encap_data = (struct esp_decap_data *) (decap->decap_data); + encap_data->proto = 0; + + switch (decap->decap_type) { + case UDP_ENCAP_ESPINUDP: + case UDP_ENCAP_ESPINUDP_NON_IKE: + encap_data->proto = AF_INET; + encap_data->saddr.a4 = iph->saddr; + encap_data->sport = uh->source; + encap_len = (void*)esph - (void*)uh; + break; + + default: + err = -ENOMEM; + goto out; + } + } + + iph->protocol = nexthdr[1]; + pskb_trim(skb, skb->len - alen - padlen - 2); + memcpy(workbuf, skb->nh.raw, iph->ihl*4); + skb->h.raw = skb_pull(skb, sizeof(struct ip_esp_hdr) + esp->conf.ivlen); + skb->nh.raw += encap_len + sizeof(struct ip_esp_hdr) + esp->conf.ivlen; + memcpy(skb->nh.raw, workbuf, iph->ihl*4); + skb->nh.iph->tot_len = htons(skb->len); + +out: + return err; +} + +#if defined CONFIG_ACRYPTO || defined CONFIG_ACRYPTO_MODULE + +struct esp4_input_priv +{ + struct sk_buff *skb; + struct xfrm_decap_state *decap; + int nfrags; + int alen; + int elen; + struct xfrm4_input_shared *sh; + + int (*callback)(struct xfrm4_input_shared *sh); + + void *free; +}; + +extern int xfrm4_run_next(struct xfrm4_input_shared *sh); +extern int xfrm4_rcv_encap_input_finish(struct xfrm4_input_shared *sh); +extern int xfrm4_resubmit_packet(struct xfrm4_input_shared *sh); +static DEFINE_SPINLOCK(esp4_input_lock); + +static void esp4_async_input_callback(struct crypto_session_initializer *ci, struct crypto_data *data) +{ + struct esp4_input_priv *p = data->priv; + struct xfrm4_input_shared *sh = p->sh; + struct esp_data *esp = (struct esp_data *)(sh->transform); + int nfrags = p->nfrags + 1 + (esp->conf.ivlen)?1:0; + int err; + struct xfrm_state *x; + struct iphdr *iph = sh->skb->nh.iph; + + sh->callback = p->callback; + + x = xfrm_state_lookup((xfrm_address_t *)&iph->daddr, sh->spi, iph->protocol, AF_INET); + if (x == NULL) + goto drop; +#if 0 + printk("%s: x=%p, seq=%08x, spi=%08x, daddr=%u.%u.%u.%u, proto=%d, refcnt=%d.\n", + __func__, x, sh->seq, sh->spi, NIPQUAD(iph->daddr), iph->protocol, atomic_read(&x->refcnt)); +#endif + xfrm_state_put(x); + + esp4_sync_input_process(p->skb, esp, p->decap, p->nfrags, p->alen, p->elen, 1, sh->xfrm_encap); + + err = xfrm4_rcv_encap_input_finish(sh); + + kfree(p->free); + if (unlikely(nfrags > ESP_NUM_FAST_SG)) + kfree(data->sg_src); + + if (err) + goto drop; + + sh->callback(sh); + xfrm4_run_next(sh); + spin_lock_bh(&esp4_input_lock); + xfrm4_resubmit_packet(sh); + spin_unlock_bh(&esp4_input_lock); + xfrm_shared_put(sh); + + return; + +drop: + xfrm_shared_put(sh); +} + +static int esp4_acrypto_process(struct sk_buff *orig_skb, struct xfrm4_input_shared *sh, + struct xfrm_decap_state *decap, int nfrags, int alen, int elen) +{ + struct crypto_data data; + struct crypto_session_initializer ci; + struct crypto_session *s; + u8 *key, *iv; + struct esp_data *esp = (struct esp_data *)(sh->transform); + struct scatterlist *sg = &esp->sgbuf[0]; + struct esp4_input_priv *p; + struct sk_buff *skb; + + nfrags++; /* key */ + + if (esp->conf.ivlen) + nfrags++; + + memset(&ci, 0, sizeof(ci)); + memset(&data, 0, sizeof(data)); + + ci.operation = CRYPTO_OP_DECRYPT; + ci.mode = crypto_tfm