From http://wpsnipp.com/index.php/cat/add-nofollow-attribute-to-category-specific-links/ Uses http://codex.wordpress.org/Function_Reference/wp_rel_nofollow
<?php
function nofollow_cat_posts($text) {
global $post;
if( in_category(1) ) { // SET CATEGORY ID HERE
$text = stripslashes(wp_rel_nofollow($text));
}
return $text;
}
add_filter('the_content', 'nofollow_cat_posts');